diff --git a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml index 6203ca29b..3ba0ebf55 100644 --- a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml +++ b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml @@ -30,6 +30,7 @@ hosts: - "{{ mon_group_name|default('mons') }}" + - "{{ mgr_group_name|default('mgrs') }}" - "{{ osd_group_name|default('osds') }}" - "{{ mds_group_name|default('mdss') }}" - "{{ rgw_group_name|default('rgws') }}" @@ -39,32 +40,6 @@ become: true tasks: - - name: install docker and dependencies for the docker module - package: - name: "{{ item }}" - state: present - with_items: - - python-docker-py - - python-urllib3 - - docker - when: ansible_os_family == 'RedHat' - - - name: install docker-py for the docker module - package: - name: "{{ item }}" - state: present - with_items: - - docker-py - - python-urllib3 - - docker - when: ansible_os_family == 'Debian' - - - name: start docker service - service: - name: docker - state: started - enabled: yes - - name: check if selinux is enabled command: getenforce register: sestatus @@ -81,6 +56,12 @@ - sestatus.stdout != 'Disabled' - ansible_os_family == 'RedHat' + - name: gather and delegate facts + setup: + delegate_to: "{{ item }}" + delegate_facts: True + with_items: "{{ groups['all'] }}" + - name: switching from non-containerized to containerized ceph mon vars: @@ -143,9 +124,11 @@ - ceph-mon post_tasks: + # We don't do a container test by running 'docker exec ...' since not all the monitors have switched to containers yet. + # Thus, we continue to use the 'ceph' binary from the host, there is no issue with that. - name: waiting for the containerized monitor to join the quorum... shell: | - docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["quorum_names"])' + ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["quorum_names"])' register: result until: "{{ ansible_hostname in result.stdout }}" retries: "{{ health_mon_check_retries }}" @@ -153,6 +136,48 @@ delegate_to: "{{ mon_host }}" +- name: switching from non-containerized to containerized ceph mgr + + hosts: + - "{{ mgr_group_name|default('mgrs') }}" + + vars: + containerized_deployment: true + mgr_group_name: mgrs + + serial: 1 + become: true + + pre_tasks: + - name: stop non-containerized ceph mgr(s) + service: + name: "ceph-mgr@{{ ansible_hostname }}" + state: stopped + enabled: no + + - set_fact: + ceph_uid: 64045 + when: ceph_docker_image_tag | match("latest") or ceph_docker_image_tag | search("ubuntu") + + - set_fact: + ceph_uid: 167 + when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph") or ceph_docker_image_tag | search("fedora") + + - name: set proper ownership on ceph directories + file: + path: "{{ item }}" + owner: "{{ ceph_uid }}" + recurse: yes + with_items: + - /var/lib/ceph + - /etc/ceph + + roles: + - ceph-defaults + - ceph-docker-common + - ceph-mgr + + - name: switching from non-containerized to containerized ceph osd vars: @@ -182,10 +207,11 @@ changed_when: false - name: stop non-containerized ceph osd(s) - service: + systemd: name: "{{ item }}" state: stopped enabled: no + masked: yes with_items: "{{ running_osds.stdout_lines }}" - set_fact: @@ -259,6 +285,10 @@ hosts: - "{{ mds_group_name|default('mdss') }}" + vars: + containerized_deployment: true + mds_group_name: mdss + serial: 1 become: true @@ -297,13 +327,17 @@ hosts: - "{{ rgw_group_name|default('rgws') }}" + vars: + containerized_deployment: true + rgw_group_name: rgws + serial: 1 become: true pre_tasks: - name: stop non-containerized ceph rgw(s) service: - name: "ceph-rgw@{{ ansible_hostname }}" + name: "ceph-radosgw@rgw.{{ ansible_hostname }}" state: stopped enabled: no @@ -335,13 +369,17 @@ hosts: - "{{ rbdmirror_group_name|default('rbdmirrors') }}" + vars: + containerized_deployment: true + rbdmirror_group_name: rbdmirrors + serial: 1 become: true pre_tasks: - name: stop non-containerized ceph rbd mirror(s) service: - name: "ceph-rbd-mirror@{{ ansible_hostname }}" + name: "ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }}" state: stopped enabled: no @@ -373,6 +411,10 @@ hosts: - "{{ nfs_group_name|default('nfss') }}" + vars: + containerized_deployment: true + nfs_group_name: nfss + serial: 1 become: true diff --git a/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml b/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml index effe2a346..e1655c1a2 100644 --- a/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml +++ b/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml @@ -1,5 +1,4 @@ --- -# To install docker on debian - name: allow apt to use a repository over https (debian) package: name: "{{ item }}" @@ -10,15 +9,11 @@ - ca-certificates - curl - software-properties-common - when: ansible_distribution == 'Debian' - tags: - with_pkg - name: add docker's gpg key apt_key: url: https://apt.dockerproject.org/gpg state: present - when: ansible_distribution == 'Debian' - name: add docker and debian testing repository apt_repository: @@ -27,7 +22,6 @@ with_items: - "deb https://apt.dockerproject.org/repo/ debian-{{ ansible_distribution_release }} main" - "deb http://http.us.debian.org/debian/ testing contrib main" - when: ansible_distribution == 'Debian' - name: install pip from testing on debian package: @@ -35,26 +29,17 @@ state: present default_release: testing update_cache: yes - when: ansible_distribution == 'Debian' - tags: - with_pkg - name: install docker-py via pip for debian pip: name: docker-py state: latest - tags: - with_pkg - when: ansible_distribution == 'Debian' - name: install docker on debian package: name: docker-engine state: present update_cache: yes - when: ansible_distribution == 'Debian' - tags: - with_pkg # NOTE (jimcurtis): need at least version 1.9.0 of six or we get: # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined @@ -62,6 +47,3 @@ pip: name: six version: 1.9.0 - when: ansible_distribution == 'Debian' - tags: - with_pkg diff --git a/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml b/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml index cb0951c86..4296d5565 100644 --- a/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml +++ b/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml @@ -1,7 +1,10 @@ --- -# Manage debian in a separate file because of specificities +- include: remove_ceph_udev_rules.yml + - include: debian_prerequisites.yml when: ansible_distribution == 'Debian' + tags: + with_pkg - name: install docker on ubuntu package: diff --git a/roles/ceph-docker-common/tasks/pre_requisites/remove_ceph_udev_rules.yml b/roles/ceph-docker-common/tasks/pre_requisites/remove_ceph_udev_rules.yml new file mode 100644 index 000000000..3e4ed3c6b --- /dev/null +++ b/roles/ceph-docker-common/tasks/pre_requisites/remove_ceph_udev_rules.yml @@ -0,0 +1,8 @@ +--- +- name: remove ceph udev rules + file: + path: "{{ item }}" + state: absent + with_items: + - /usr/lib/udev/rules.d/95-ceph-osd.rules + - /usr/lib/udev/rules.d/60-ceph-by-parttypeuuid.rules diff --git a/roles/ceph-docker-common/tasks/stat_ceph_files.yml b/roles/ceph-docker-common/tasks/stat_ceph_files.yml index 4492843c2..01573183c 100644 --- a/roles/ceph-docker-common/tasks/stat_ceph_files.yml +++ b/roles/ceph-docker-common/tasks/stat_ceph_files.yml @@ -25,17 +25,17 @@ tmp_ceph_mgr_keys: "{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring" with_items: "{{ groups.get(mgr_group_name, []) }}" register: tmp_ceph_mgr_keys_result - when: "{{ groups.get(mgr_group_name, []) | length > 0 }}" + when: groups.get(mgr_group_name, []) | length > 0 - name: convert mgr keys to an array set_fact: ceph_mgr_keys: "{{ tmp_ceph_mgr_keys_result.results | map(attribute='ansible_facts.tmp_ceph_mgr_keys') | list }}" - when: "{{ groups.get(mgr_group_name, []) | length > 0 }}" + when: groups.get(mgr_group_name, []) | length > 0 - name: merge mgr keys to config and keys paths set_fact: ceph_config_keys: "{{ ceph_config_keys + ceph_mgr_keys }}" - when: "{{ groups.get(mgr_group_name, []) | length > 0 }}" + when: groups.get(mgr_group_name, []) | length > 0 - name: stat for ceph config and keys local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }} diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index 626ecbc21..b3c46f55c 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -8,25 +8,65 @@ delay: 15 until: monitor_socket.rc == 0 -- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[item]['ansible_' + monitor_interface].ipv4.address }} - with_items: "{{ groups[mon_group_name] }}" +- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_interface + command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['ansible_' + monitor_interface].ipv4.address }} changed_when: false failed_when: false when: - - inventory_hostname == groups[mon_group_name][0] - not containerized_deployment_with_kv - ip_version == 'ipv4' + - hostvars[groups[mon_group_name][0]]['monitor_interface'] is defined + - hostvars[groups[mon_group_name][0]]['monitor_interface'] != 'interface' -- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[item]['ansible_' + monitor_interface].ipv6[0].address }}] - with_items: "{{ groups[mon_group_name] }}" +- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address + command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['monitor_address'] }} + changed_when: false + failed_when: false + when: + - not containerized_deployment_with_kv + - ip_version == 'ipv4' + - hostvars[groups[mon_group_name][0]]['monitor_address'] is defined + - hostvars[groups[mon_group_name][0]]['monitor_address'] != '0.0.0.0' + +- name: ipv4 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address_block + command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ hostvars[groups[mon_group_name][0]]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }} + changed_when: false + failed_when: false + when: + - not containerized_deployment_with_kv + - ip_version == 'ipv4' + - hostvars[groups[mon_group_name][0]]['monitor_address_block'] is defined + - hostvars[groups[mon_group_name][0]]['monitor_address_block'] | length > 0 + +- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_interface + command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['ansible_' + monitor_interface].ipv6[0].address }}] changed_when: false failed_when: false when: - - inventory_hostname == groups[mon_group_name][0] - not containerized_deployment_with_kv - ip_version == 'ipv6' + - hostvars[groups[mon_group_name][0]]['monitor_interface'] is defined + - hostvars[groups[mon_group_name][0]]['monitor_interface'] != 'interface' + +- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address + command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['monitor_address'] }}] + changed_when: false + failed_when: false + when: + - not containerized_deployment_with_kv + - ip_version == 'ipv6' + - hostvars[groups[mon_group_name][0]]['monitor_address'] is defined + - hostvars[groups[mon_group_name][0]]['monitor_address'] != '0.0.0.0' + +- name: ipv6 - force peer addition as potential bootstrap peer for cluster bringup - monitor_address_block + command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint [{{ hostvars[groups[mon_group_name][0]]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }}] + changed_when: false + failed_when: false + when: + - not containerized_deployment_with_kv + - ip_version == 'ipv6' + - hostvars[groups[mon_group_name][0]]['monitor_address_block'] is defined + - hostvars[groups[mon_group_name][0]]['monitor_address_block'] | length > 0 - include: copy_configs.yml when: not containerized_deployment_with_kv diff --git a/site-docker.yml.sample b/site-docker.yml.sample index 238997aea..29272223c 100644 --- a/site-docker.yml.sample +++ b/site-docker.yml.sample @@ -14,6 +14,8 @@ - iscsigws - mgrs + gather_facts: false + tasks: - name: gather and delegate facts setup: @@ -33,6 +35,7 @@ - hosts: osds become: True + gather_facts: false roles: - ceph-defaults - ceph-docker-common @@ -41,6 +44,7 @@ - hosts: mdss become: True + gather_facts: false roles: - ceph-defaults - ceph-docker-common @@ -49,6 +53,7 @@ - hosts: rgws become: True + gather_facts: false roles: - ceph-defaults - ceph-docker-common @@ -57,6 +62,7 @@ - hosts: nfss become: True + gather_facts: false roles: - ceph-defaults - ceph-docker-common @@ -65,6 +71,7 @@ - hosts: rbd_mirrors become: True + gather_facts: false roles: - ceph-defaults - ceph-docker-common @@ -73,6 +80,7 @@ - hosts: restapis become: True + gather_facts: false roles: - ceph-defaults - ceph-docker-common @@ -81,6 +89,7 @@ - hosts: mgrs become: True + gather_facts: false roles: - { role: ceph-defaults, when: "ceph_release_num.{{ ceph_stable_release }} > ceph_release_num.jewel" } - { role: ceph-docker-common, when: "ceph_release_num.{{ ceph_stable_release }} > ceph_release_num.jewel" } @@ -89,6 +98,7 @@ - hosts: clients become: True + gather_facts: false roles: - ceph-defaults - ceph-common diff --git a/tests/functional/centos/7/cluster/hosts-switch-to-containers b/tests/functional/centos/7/cluster/hosts-switch-to-containers new file mode 100644 index 000000000..85f592c42 --- /dev/null +++ b/tests/functional/centos/7/cluster/hosts-switch-to-containers @@ -0,0 +1,22 @@ +[all:vars] +docker=True + +[mons] +ceph-mon0 monitor_address=192.168.1.10 +ceph-mon1 monitor_interface=eth1 +ceph-mon2 monitor_address=192.168.1.12 + +[osds] +ceph-osd0 + +[mdss] +ceph-mds0 + +[rgws] +ceph-rgw0 + +[clients] +ceph-client0 + +[mgrs] +ceph-mgr0 diff --git a/tox.ini b/tox.ini index 6b0c0deac..aa4cf3a15 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {dev,jewel,luminous,rhcs}-{ansible2.2,ansible2.3}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster,cluster,purge_docker_cluster,update_docker_cluster} +envlist = {dev,jewel,luminous,rhcs}-{ansible2.2,ansible2.3}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster,cluster,purge_docker_cluster,update_docker_cluster,switch_to_containers} {dev,luminous}-{ansible2.2,ansible2.3}-{bluestore_journal_collocation,bluestore_cluster,bluestore_dmcrypt_journal,bluestore_dmcrypt_journal_collocation,bluestore_docker_cluster,bluestore_docker_dedicated_journal,bluestore_docker_dmcrypt_journal_collocation,lvm_osds,purge_lvm_osds,shrink_mon,shrink_osd} skipsdist = True @@ -96,6 +96,21 @@ commands= osd_to_kill=0 \ " +[switch-to-containers] +commands= + cp {toxinidir}/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml {toxinidir}/switch-from-non-containerized-to-containerized-ceph-daemons.yml + ansible-playbook -vv -i {changedir}/hosts {toxinidir}/switch-from-non-containerized-to-containerized-ceph-daemons.yml --extra-vars "\ + ireallymeanit=yes \ + fetch_directory={env:FETCH_DIRECTORY:{changedir}/fetch} \ + ceph_docker_registry={env:CEPH_DOCKER_REGISTRY:docker.io} \ + ceph_docker_image={env:UPDATE_CEPH_DOCKER_IMAGE:ceph/daemon} \ + ceph_docker_image_tag={env:UPDATE_CEPH_DOCKER_IMAGE_TAG:latest} \ + ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} \ + ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \ + " + + testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-switch-to-containers {toxinidir}/tests/functional/tests + [testenv] whitelist_externals = vagrant @@ -157,6 +172,7 @@ changedir= purge_dmcrypt: {toxinidir}/tests/functional/centos/7/crypt-ded-jrn update_dmcrypt: {toxinidir}/tests/functional/centos/7/crypt-ded-jrn update_cluster: {toxinidir}/tests/functional/centos/7/cluster + switch_to_containers: {toxinidir}/tests/functional/centos/7/cluster bluestore_journal_collocation: {toxinidir}/tests/functional/centos/7/bs-jrn-col bluestore_cluster: {toxinidir}/tests/functional/centos/7/bluestore bluestore_dmcrypt_journal: {toxinidir}/tests/functional/centos/7/bs-crypt-ded-jrn @@ -197,6 +213,7 @@ commands= purge_lvm_osds: {[purge-lvm]commands} purge_dmcrypt: {[purge]commands} purge_docker_cluster: {[purge]commands} + switch_to_containers: {[switch-to-containers]commands} update_dmcrypt: {[update]commands} update_cluster: {[update]commands} update_docker_cluster: {[update]commands}