switch-from-non-containerized-to-containerized: mask unit files

We must mask the image so we are sure that even if the system reboots
then the OSDs won't start.

Also remove Ceph udev rules if found on the system prior to deploy
containers. If we don't do this we are exposed to conflicts between udev
rules and sytemd unit files.

Also add the CI will now test the migration from a non-containerized cluster to a
containerized cluster.

Signed-off-by: Sébastien Han <seb@redhat.com>

infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml

@@ -32,6 +32,7 @@
     - "{{ mon_group_name|default('mons') }}"
     - "{{ osd_group_name|default('osds') }}"
     - "{{ mds_group_name|default('mdss') }}"
+    - "{{ mgr_group_name|default('mgrs') }}"
     - "{{ rgw_group_name|default('rgws') }}"
     - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
     - "{{ nfs_group_name|default('nfss') }}"
@@ -39,32 +40,6 @@
   become: true
 
   tasks:
-    - name: install docker and dependencies for the docker module
-      package:
-        name: "{{ item }}"
-        state: present
-      with_items:
-        - python-docker-py
-        - python-urllib3
-        - docker
-      when: ansible_os_family == 'RedHat'
-
-    - name: install docker-py for the docker module
-      package:
-        name: "{{ item }}"
-        state: present
-      with_items:
-        - docker-py
-        - python-urllib3
-        - docker
-      when: ansible_os_family == 'Debian'
-
-    - name: start docker service
-      service:
-        name: docker
-        state: started
-        enabled: yes
-
     - name: check if selinux is enabled
       command: getenforce
       register: sestatus
@@ -81,6 +56,12 @@
         - sestatus.stdout != 'Disabled'
         - ansible_os_family == 'RedHat'
 
+    - name: gather and delegate facts
+      setup:
+      delegate_to: "{{ item }}"
+      delegate_facts: True
+      with_items: "{{ groups['all'] }}"
+
 - name: switching from non-containerized to containerized ceph mon
 
   vars:
@@ -143,9 +124,11 @@
     - ceph-mon
 
   post_tasks:
+    # We don't do a container test by running 'docker exec ...' since not all the monitors have switched to containers yet.
+    # Thus, we continue to use the 'ceph' binary from the host, there is no issue with that.
     - name: waiting for the containerized monitor to join the quorum...
       shell: |
-        docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["quorum_names"])'
+        ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["quorum_names"])'
       register: result
       until: "{{ ansible_hostname in result.stdout }}"
       retries: "{{ health_mon_check_retries }}"
@@ -182,10 +165,11 @@
       changed_when: false
 
     - name: stop non-containerized ceph osd(s)
-      service:
+      systemd:
         name: "{{ item }}"
         state: stopped
         enabled: no
+        masked: yes
       with_items: "{{ running_osds.stdout_lines }}"
 
     - set_fact:
@@ -259,6 +243,10 @@
   hosts:
     - "{{ mds_group_name|default('mdss') }}"
 
+  vars:
+    containerized_deployment: true
+    mds_group_name: mdss
+
   serial: 1
   become: true
 
@@ -297,13 +285,17 @@
   hosts:
     - "{{ rgw_group_name|default('rgws') }}"
 
+  vars:
+    containerized_deployment: true
+    rgw_group_name: rgws
+
   serial: 1
   become: true
 
   pre_tasks:
     - name: stop non-containerized ceph rgw(s)
       service:
-        name: "ceph-rgw@{{ ansible_hostname }}"
+        name: "ceph-radosgw@rgw.{{ ansible_hostname }}"
         state: stopped
         enabled: no
 
@@ -335,13 +327,17 @@
   hosts:
     - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
 
+  vars:
+    containerized_deployment: true
+    rbdmirror_group_name: rbdmirrors
+
   serial: 1
   become: true
 
   pre_tasks:
     - name: stop non-containerized ceph rbd mirror(s)
       service:
-        name: "ceph-rbd-mirror@{{ ansible_hostname }}"
+        name: "ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }}"
         state: stopped
         enabled: no
 
@@ -373,6 +369,10 @@
   hosts:
     - "{{ nfs_group_name|default('nfss') }}"
 
+  vars:
+    containerized_deployment: true
+    nfs_group_name: nfss
+
   serial: 1
   become: true
 

roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml

@@ -1,5 +1,4 @@
 ---
-# To install docker on debian
 - name: allow apt to use a repository over https (debian)
   package:
     name: "{{ item }}"
@@ -10,15 +9,11 @@
     - ca-certificates
     - curl
     - software-properties-common
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
 
 - name: add docker's gpg key
   apt_key:
     url: https://apt.dockerproject.org/gpg
     state: present
-  when: ansible_distribution == 'Debian'
 
 - name: add docker and debian testing repository
   apt_repository:
@@ -27,7 +22,6 @@
   with_items:
     - "deb https://apt.dockerproject.org/repo/ debian-{{ ansible_distribution_release }} main"
     - "deb http://http.us.debian.org/debian/ testing contrib main"
-  when: ansible_distribution == 'Debian'
 
 - name: install pip from testing on debian
   package:
@@ -35,26 +29,17 @@
     state: present
     default_release: testing
     update_cache: yes
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
 
 - name: install docker-py via pip for debian
   pip:
     name: docker-py
     state: latest
-  tags:
-    with_pkg
-  when: ansible_distribution == 'Debian'
 
 - name: install docker on debian
   package:
     name: docker-engine
     state: present
     update_cache: yes
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
 
 # NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
 # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
@@ -62,6 +47,3 @@
   pip:
     name: six
     version: 1.9.0
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg

roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml

@@ -1,7 +1,10 @@
 ---
-# Manage debian in a separate file because of specificities
+- include: remove_ceph_udev_rules.yml
+
 - include: debian_prerequisites.yml
   when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
 
 - name: install docker on ubuntu
   package:

roles/ceph-docker-common/tasks/pre_requisites/remove_ceph_udev_rules.yml

@@ -0,0 +1,8 @@
+---
+- name: remove ceph udev rules
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /usr/lib/udev/rules.d/95-ceph-osd.rules
+    - /usr/lib/udev/rules.d/60-ceph-by-parttypeuuid.rules

roles/ceph-docker-common/tasks/stat_ceph_files.yml

@@ -25,17 +25,17 @@
     tmp_ceph_mgr_keys: "{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
   with_items: "{{ groups.get(mgr_group_name, []) }}"
   register: tmp_ceph_mgr_keys_result
-  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+  when: groups.get(mgr_group_name, []) | length > 0
 
 - name: convert mgr keys to an array
   set_fact:
     ceph_mgr_keys: "{{ tmp_ceph_mgr_keys_result.results | map(attribute='ansible_facts.tmp_ceph_mgr_keys') | list }}"
-  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+  when: groups.get(mgr_group_name, []) | length > 0
 
 - name: merge mgr keys to config and keys paths
   set_fact:
     ceph_config_keys: "{{ ceph_config_keys + ceph_mgr_keys }}"
-  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+  when: groups.get(mgr_group_name, []) | length > 0
 
 - name: stat for ceph config and keys
   local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}

roles/ceph-mon/tasks/docker/main.yml

@@ -8,25 +8,65 @@
   delay: 15
   until: monitor_socket.rc == 0
 
-- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup
-  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[item]['ansible_' + monitor_interface].ipv4.address }}
-  with_items: "{{ groups[mon_group_name] }}"
+- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_interface
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['ansible_' + monitor_interface].ipv4.address }}
   changed_when: false
   failed_when: false
   when:
-    - inventory_hostname == groups[mon_group_name][0]
     - not containerized_deployment_with_kv
     - ip_version == 'ipv4'
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] != 'interface'
 
-- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup
-  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[item]['ansible_' + monitor_interface].ipv6[0].address }}]
-  with_items: "{{ groups[mon_group_name] }}"
+- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['monitor_address'] }}
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv4'
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] != '0.0.0.0'
+
+- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address_block
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }}
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv4'
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] | length > 0
+
+- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_interface
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['ansible_' + monitor_interface].ipv6[0].address }}]
   changed_when: false
   failed_when: false
   when:
-    - inventory_hostname == groups[mon_group_name][0]
     - not containerized_deployment_with_kv
     - ip_version == 'ipv6'
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] != 'interface'
+
+- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['monitor_address'] }}]
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv6'
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] != '0.0.0.0'
+
+- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address_block
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }}]
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv6'
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] | length > 0
 
 - include: copy_configs.yml
   when: not containerized_deployment_with_kv

site-docker.yml.sample

@@ -14,6 +14,8 @@
   - iscsigws
   - mgrs
 
+  gather_facts: false
+
   tasks:
     - name: gather and delegate facts
       setup:
@@ -33,6 +35,7 @@
 
 - hosts: osds
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -41,6 +44,7 @@
 
 - hosts: mdss
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -49,6 +53,7 @@
 
 - hosts: rgws
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -57,6 +62,7 @@
 
 - hosts: nfss
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -65,6 +71,7 @@
 
 - hosts: rbd_mirrors
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -73,6 +80,7 @@
 
 - hosts: restapis
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -81,6 +89,7 @@
 
 - hosts: mgrs
   become: True
+  gather_facts: false
   roles:
     - { role: ceph-defaults, when: "ceph_release_num.{{ ceph_stable_release }} > ceph_release_num.jewel" }
     - { role: ceph-docker-common, when: "ceph_release_num.{{ ceph_stable_release }} > ceph_release_num.jewel" }
@@ -89,6 +98,7 @@
 
 - hosts: clients
   become: True
+  gather_facts: false
   roles:
   - ceph-defaults
   - ceph-common

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist = {dev,jewel,luminous,rhcs}-{ansible2.2,ansible2.3}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster,cluster,purge_docker_cluster,update_docker_cluster}
-  {dev,luminous}-{ansible2.2,ansible2.3}-{bluestore_journal_collocation,bluestore_cluster,bluestore_dmcrypt_journal,bluestore_dmcrypt_journal_collocation,bluestore_docker_cluster,bluestore_docker_dedicated_journal,bluestore_docker_dmcrypt_journal_collocation,lvm_osds,purge_lvm_osds,shrink_mon,shrink_osd}
+  {dev,luminous}-{ansible2.2,ansible2.3}-{bluestore_journal_collocation,bluestore_cluster,bluestore_dmcrypt_journal,bluestore_dmcrypt_journal_collocation,bluestore_docker_cluster,bluestore_docker_dedicated_journal,bluestore_docker_dmcrypt_journal_collocation,lvm_osds,purge_lvm_osds,shrink_mon,shrink_osd,switch_to_containers}
 
 skipsdist = True
 
@@ -96,12 +96,29 @@ commands=
       osd_to_kill=0 \
   "
 
+[switch-to-containers]
+commands=
+  cp {toxinidir}/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml {toxinidir}/switch-from-non-containerized-to-containerized-ceph-daemons.yml
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/switch-from-non-containerized-to-containerized-ceph-daemons.yml --extra-vars "\
+      ireallymeanit=yes \
+      fetch_directory={env:FETCH_DIRECTORY:{changedir}/fetch} \
+      ceph_docker_registry={env:CEPH_DOCKER_REGISTRY:docker.io} \
+      ceph_docker_image={env:UPDATE_CEPH_DOCKER_IMAGE:ceph/daemon} \
+      ceph_docker_image_tag={env:UPDATE_CEPH_DOCKER_IMAGE_TAG:latest} \
+      ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} \
+      ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \
+  "
+
+  echo "docker: True" >> {changedir}/group_vars/all
+  testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
+
 [testenv]
 whitelist_externals =
     vagrant
     bash
     pip
     cp
+    echo
 passenv=*
 sitepackages=True
 setenv=
@@ -157,6 +174,7 @@ changedir=
   purge_dmcrypt: {toxinidir}/tests/functional/centos/7/crypt-ded-jrn
   update_dmcrypt: {toxinidir}/tests/functional/centos/7/crypt-ded-jrn
   update_cluster: {toxinidir}/tests/functional/centos/7/cluster
+  switch_to_containers: {toxinidir}/tests/functional/centos/7/cluster
   bluestore_journal_collocation: {toxinidir}/tests/functional/centos/7/bs-jrn-col
   bluestore_cluster: {toxinidir}/tests/functional/centos/7/bluestore
   bluestore_dmcrypt_journal: {toxinidir}/tests/functional/centos/7/bs-crypt-ded-jrn
@@ -197,6 +215,7 @@ commands=
   purge_lvm_osds: {[purge-lvm]commands}
   purge_dmcrypt: {[purge]commands}
   purge_docker_cluster: {[purge]commands}
+  switch_to_containers: {[switch-to-containers]commands}
   update_dmcrypt: {[update]commands}
   update_cluster: {[update]commands}
   update_docker_cluster: {[update]commands}