Merge pull request #1812 from ceph/switch-migration-conta

switch-from-non-containerized-to-containerized: mask unit files

infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml

@@ -30,6 +30,7 @@
 
   hosts:
     - "{{ mon_group_name|default('mons') }}"
+    - "{{ mgr_group_name|default('mgrs') }}"
     - "{{ osd_group_name|default('osds') }}"
     - "{{ mds_group_name|default('mdss') }}"
     - "{{ rgw_group_name|default('rgws') }}"
@@ -39,32 +40,6 @@
   become: true
 
   tasks:
-    - name: install docker and dependencies for the docker module
-      package:
-        name: "{{ item }}"
-        state: present
-      with_items:
-        - python-docker-py
-        - python-urllib3
-        - docker
-      when: ansible_os_family == 'RedHat'
-
-    - name: install docker-py for the docker module
-      package:
-        name: "{{ item }}"
-        state: present
-      with_items:
-        - docker-py
-        - python-urllib3
-        - docker
-      when: ansible_os_family == 'Debian'
-
-    - name: start docker service
-      service:
-        name: docker
-        state: started
-        enabled: yes
-
     - name: check if selinux is enabled
       command: getenforce
       register: sestatus
@@ -81,6 +56,12 @@
         - sestatus.stdout != 'Disabled'
         - ansible_os_family == 'RedHat'
 
+    - name: gather and delegate facts
+      setup:
+      delegate_to: "{{ item }}"
+      delegate_facts: True
+      with_items: "{{ groups['all'] }}"
+
 - name: switching from non-containerized to containerized ceph mon
 
   vars:
@@ -143,9 +124,11 @@
     - ceph-mon
 
   post_tasks:
+    # We don't do a container test by running 'docker exec ...' since not all the monitors have switched to containers yet.
+    # Thus, we continue to use the 'ceph' binary from the host, there is no issue with that.
     - name: waiting for the containerized monitor to join the quorum...
       shell: |
-        docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["quorum_names"])'
+        ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["quorum_names"])'
       register: result
       until: "{{ ansible_hostname in result.stdout }}"
       retries: "{{ health_mon_check_retries }}"
@@ -153,6 +136,48 @@
       delegate_to: "{{ mon_host }}"
 
 
+- name: switching from non-containerized to containerized ceph mgr
+
+  hosts:
+    - "{{ mgr_group_name|default('mgrs') }}"
+
+  vars:
+    containerized_deployment: true
+    mgr_group_name: mgrs
+
+  serial: 1
+  become: true
+
+  pre_tasks:
+    - name: stop non-containerized ceph mgr(s)
+      service:
+        name: "ceph-mgr@{{ ansible_hostname }}"
+        state: stopped
+        enabled: no
+
+    - set_fact:
+        ceph_uid: 64045
+      when: ceph_docker_image_tag | match("latest") or ceph_docker_image_tag | search("ubuntu")
+
+    - set_fact:
+        ceph_uid: 167
+      when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph") or ceph_docker_image_tag | search("fedora")
+
+    - name: set proper ownership on ceph directories
+      file:
+        path: "{{ item }}"
+        owner: "{{ ceph_uid }}"
+        recurse: yes
+      with_items:
+        - /var/lib/ceph
+        - /etc/ceph
+
+  roles:
+    - ceph-defaults
+    - ceph-docker-common
+    - ceph-mgr
+
+
 - name: switching from non-containerized to containerized ceph osd
 
   vars:
@@ -182,10 +207,11 @@
       changed_when: false
 
     - name: stop non-containerized ceph osd(s)
-      service:
+      systemd:
         name: "{{ item }}"
         state: stopped
         enabled: no
+        masked: yes
       with_items: "{{ running_osds.stdout_lines }}"
 
     - set_fact:
@@ -259,6 +285,10 @@
   hosts:
     - "{{ mds_group_name|default('mdss') }}"
 
+  vars:
+    containerized_deployment: true
+    mds_group_name: mdss
+
   serial: 1
   become: true
 
@@ -297,13 +327,17 @@
   hosts:
     - "{{ rgw_group_name|default('rgws') }}"
 
+  vars:
+    containerized_deployment: true
+    rgw_group_name: rgws
+
   serial: 1
   become: true
 
   pre_tasks:
     - name: stop non-containerized ceph rgw(s)
       service:
-        name: "ceph-rgw@{{ ansible_hostname }}"
+        name: "ceph-radosgw@rgw.{{ ansible_hostname }}"
         state: stopped
         enabled: no
 
@@ -335,13 +369,17 @@
   hosts:
     - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
 
+  vars:
+    containerized_deployment: true
+    rbdmirror_group_name: rbdmirrors
+
   serial: 1
   become: true
 
   pre_tasks:
     - name: stop non-containerized ceph rbd mirror(s)
       service:
-        name: "ceph-rbd-mirror@{{ ansible_hostname }}"
+        name: "ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }}"
         state: stopped
         enabled: no
 
@@ -373,6 +411,10 @@
   hosts:
     - "{{ nfs_group_name|default('nfss') }}"
 
+  vars:
+    containerized_deployment: true
+    nfs_group_name: nfss
+
   serial: 1
   become: true
 

roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml

@@ -1,5 +1,4 @@
 ---
-# To install docker on debian
 - name: allow apt to use a repository over https (debian)
   package:
     name: "{{ item }}"
@@ -10,15 +9,11 @@
     - ca-certificates
     - curl
     - software-properties-common
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
 
 - name: add docker's gpg key
   apt_key:
     url: https://apt.dockerproject.org/gpg
     state: present
-  when: ansible_distribution == 'Debian'
 
 - name: add docker and debian testing repository
   apt_repository:
@@ -27,7 +22,6 @@
   with_items:
     - "deb https://apt.dockerproject.org/repo/ debian-{{ ansible_distribution_release }} main"
     - "deb http://http.us.debian.org/debian/ testing contrib main"
-  when: ansible_distribution == 'Debian'
 
 - name: install pip from testing on debian
   package:
@@ -35,26 +29,17 @@
     state: present
     default_release: testing
     update_cache: yes
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
 
 - name: install docker-py via pip for debian
   pip:
     name: docker-py
     state: latest
-  tags:
-    with_pkg
-  when: ansible_distribution == 'Debian'
 
 - name: install docker on debian
   package:
     name: docker-engine
     state: present
     update_cache: yes
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
 
 # NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
 # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
@@ -62,6 +47,3 @@
   pip:
     name: six
     version: 1.9.0
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg

roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml

@@ -1,7 +1,10 @@
 ---
-# Manage debian in a separate file because of specificities
+- include: remove_ceph_udev_rules.yml
+
 - include: debian_prerequisites.yml
   when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
 
 - name: install docker on ubuntu
   package:

roles/ceph-docker-common/tasks/pre_requisites/remove_ceph_udev_rules.yml

@@ -0,0 +1,8 @@
+---
+- name: remove ceph udev rules
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /usr/lib/udev/rules.d/95-ceph-osd.rules
+    - /usr/lib/udev/rules.d/60-ceph-by-parttypeuuid.rules

roles/ceph-docker-common/tasks/stat_ceph_files.yml

@@ -25,17 +25,17 @@
     tmp_ceph_mgr_keys: "{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
   with_items: "{{ groups.get(mgr_group_name, []) }}"
   register: tmp_ceph_mgr_keys_result
-  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+  when: groups.get(mgr_group_name, []) | length > 0
 
 - name: convert mgr keys to an array
   set_fact:
     ceph_mgr_keys: "{{ tmp_ceph_mgr_keys_result.results | map(attribute='ansible_facts.tmp_ceph_mgr_keys') | list }}"
-  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+  when: groups.get(mgr_group_name, []) | length > 0
 
 - name: merge mgr keys to config and keys paths
   set_fact:
     ceph_config_keys: "{{ ceph_config_keys + ceph_mgr_keys }}"
-  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+  when: groups.get(mgr_group_name, []) | length > 0
 
 - name: stat for ceph config and keys
   local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}

roles/ceph-mon/tasks/docker/main.yml

@@ -8,25 +8,65 @@
   delay: 15
   until: monitor_socket.rc == 0
 
-- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup
-  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[item]['ansible_' + monitor_interface].ipv4.address }}
-  with_items: "{{ groups[mon_group_name] }}"
+- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_interface
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['ansible_' + monitor_interface].ipv4.address }}
   changed_when: false
   failed_when: false
   when:
-    - inventory_hostname == groups[mon_group_name][0]
     - not containerized_deployment_with_kv
     - ip_version == 'ipv4'
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] != 'interface'
 
-- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup
-  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[item]['ansible_' + monitor_interface].ipv6[0].address }}]
-  with_items: "{{ groups[mon_group_name] }}"
+- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['monitor_address'] }}
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv4'
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] != '0.0.0.0'
+
+- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address_block
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }}
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv4'
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] | length > 0
+
+- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_interface
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['ansible_' + monitor_interface].ipv6[0].address }}]
   changed_when: false
   failed_when: false
   when:
-    - inventory_hostname == groups[mon_group_name][0]
     - not containerized_deployment_with_kv
     - ip_version == 'ipv6'
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_interface'] != 'interface'
+
+- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['monitor_address'] }}]
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv6'
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address'] != '0.0.0.0'
+
+- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address_block
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }}]
+  changed_when: false
+  failed_when: false
+  when:
+    - not containerized_deployment_with_kv
+    - ip_version == 'ipv6'
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] is defined
+    - hostvars[groups[mon_group_name][0]]['monitor_address_block'] | length > 0
 
 - include: copy_configs.yml
   when: not containerized_deployment_with_kv

site-docker.yml.sample

@@ -14,6 +14,8 @@
   - iscsigws
   - mgrs
 
+  gather_facts: false
+
   tasks:
     - name: gather and delegate facts
       setup:
@@ -33,6 +35,7 @@
 
 - hosts: osds
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -41,6 +44,7 @@
 
 - hosts: mdss
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -49,6 +53,7 @@
 
 - hosts: rgws
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -57,6 +62,7 @@
 
 - hosts: nfss
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -65,6 +71,7 @@
 
 - hosts: rbd_mirrors
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -73,6 +80,7 @@
 
 - hosts: restapis
   become: True
+  gather_facts: false
   roles:
     - ceph-defaults
     - ceph-docker-common
@@ -81,6 +89,7 @@
 
 - hosts: mgrs
   become: True
+  gather_facts: false
   roles:
     - { role: ceph-defaults, when: "ceph_release_num.{{ ceph_stable_release }} > ceph_release_num.jewel" }
     - { role: ceph-docker-common, when: "ceph_release_num.{{ ceph_stable_release }} > ceph_release_num.jewel" }
@@ -89,6 +98,7 @@
 
 - hosts: clients
   become: True
+  gather_facts: false
   roles:
   - ceph-defaults
   - ceph-common

tests/functional/centos/7/cluster/hosts-switch-to-containers

@@ -0,0 +1,22 @@
+[all:vars]
+docker=True
+
+[mons]
+ceph-mon0 monitor_address=192.168.1.10
+ceph-mon1 monitor_interface=eth1
+ceph-mon2 monitor_address=192.168.1.12
+
+[osds]
+ceph-osd0
+
+[mdss]
+ceph-mds0
+
+[rgws]
+ceph-rgw0
+
+[clients]
+ceph-client0
+
+[mgrs]
+ceph-mgr0

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {dev,jewel,luminous,rhcs}-{ansible2.2,ansible2.3}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster,cluster,purge_docker_cluster,update_docker_cluster}
+envlist = {dev,jewel,luminous,rhcs}-{ansible2.2,ansible2.3}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster,cluster,purge_docker_cluster,update_docker_cluster,switch_to_containers}
   {dev,luminous}-{ansible2.2,ansible2.3}-{bluestore_journal_collocation,bluestore_cluster,bluestore_dmcrypt_journal,bluestore_dmcrypt_journal_collocation,bluestore_docker_cluster,bluestore_docker_dedicated_journal,bluestore_docker_dmcrypt_journal_collocation,lvm_osds,purge_lvm_osds,shrink_mon,shrink_osd}
 
 skipsdist = True
@@ -96,6 +96,21 @@ commands=
       osd_to_kill=0 \
   "
 
+[switch-to-containers]
+commands=
+  cp {toxinidir}/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml {toxinidir}/switch-from-non-containerized-to-containerized-ceph-daemons.yml
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/switch-from-non-containerized-to-containerized-ceph-daemons.yml --extra-vars "\
+      ireallymeanit=yes \
+      fetch_directory={env:FETCH_DIRECTORY:{changedir}/fetch} \
+      ceph_docker_registry={env:CEPH_DOCKER_REGISTRY:docker.io} \
+      ceph_docker_image={env:UPDATE_CEPH_DOCKER_IMAGE:ceph/daemon} \
+      ceph_docker_image_tag={env:UPDATE_CEPH_DOCKER_IMAGE_TAG:latest} \
+      ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} \
+      ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \
+  "
+
+  testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-switch-to-containers {toxinidir}/tests/functional/tests
+
 [testenv]
 whitelist_externals =
     vagrant
@@ -157,6 +172,7 @@ changedir=
   purge_dmcrypt: {toxinidir}/tests/functional/centos/7/crypt-ded-jrn
   update_dmcrypt: {toxinidir}/tests/functional/centos/7/crypt-ded-jrn
   update_cluster: {toxinidir}/tests/functional/centos/7/cluster
+  switch_to_containers: {toxinidir}/tests/functional/centos/7/cluster
   bluestore_journal_collocation: {toxinidir}/tests/functional/centos/7/bs-jrn-col
   bluestore_cluster: {toxinidir}/tests/functional/centos/7/bluestore
   bluestore_dmcrypt_journal: {toxinidir}/tests/functional/centos/7/bs-crypt-ded-jrn
@@ -197,6 +213,7 @@ commands=
   purge_lvm_osds: {[purge-lvm]commands}
   purge_dmcrypt: {[purge]commands}
   purge_docker_cluster: {[purge]commands}
+  switch_to_containers: {[switch-to-containers]commands}
   update_dmcrypt: {[update]commands}
   update_cluster: {[update]commands}
   update_docker_cluster: {[update]commands}