From 5bfa1b0d24ef91d9fe1c7368a9003c719e336f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 14 Sep 2016 17:53:30 +0200 Subject: [PATCH] ability to switch from bare metal to containerized daemons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sébastien Han --- ...inerized-to-containerized-ceph-daemons.yml | 527 ++++++++++++++++++ 1 file changed, 527 insertions(+) create mode 100644 infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml diff --git a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml new file mode 100644 index 000000000..e6085bc32 --- /dev/null +++ b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml @@ -0,0 +1,527 @@ +--- +# This playbook switches from non-containerized to containerized Ceph daemons + +- name: confirm whether user really meant to switch from non-containerized to containerized ceph daemons + + hosts: + - localhost + + gather_facts: false + + vars_prompt: + - name: ireallymeanit + prompt: Are you sure you want to switch from non-containerized to containerized ceph daemons? + default: 'no' + private: no + + tasks: + - name: exit playbook, if user did not mean to switch from non-containerized to containerized daemons? + fail: + msg: > + "Exiting switch-from-non-containerized-to-containerized-ceph-daemons.yml playbook, + cluster did not switch from non-containerized to containerized ceph daemons. + To switch from non-containerized to containerized ceph daemons, either say 'yes' on the prompt or + or use `-e ireallymeanit=yes` on the command line when + invoking the playbook" + when: ireallymeanit != 'yes' + + +- name: make sure docker is present and started + + vars: + mon_group_name: mons + osd_group_name: osds + mds_group_name: mdss + rgw_group_name: rgws + rbdmirror_group_name: rbd_mirrors + nfs_group_name: nfss + + hosts: + - "{{ mon_group_name }}" + - "{{ osd_group_name }}" + - "{{ mds_group_name }}" + - "{{ rgw_group_name }}" + - "{{ rbdmirror_group_name }}" + - "{{ nfs_group_name }}" + + become: true + + tasks: + + - name: install docker and dependancies for the docker module + package: + name: "{{ item }}" + state: present + with_items: + - python-docker-py + - python-urllib3 + - docker + when: ansible_os_family == 'RedHat' + + - name: install docker-py for the docker module + package: + name: "{{ item }}" + state: present + with_items: + - docker-py + - python-urllib3 + - docker + when: ansible_os_family == 'Debian' + + - name: start docker service + service: + name: docker + state: started + enabled: yes + + - name: check if selinux is enabled + command: getenforce + register: sestatus + changed_when: false + when: ansible_os_family == 'RedHat' + + - name: set selinux permissions + command: chcon -Rt svirt_sandbox_file_t "{{ item }}" + with_items: + - /etc/ceph + - /var/lib/ceph + changed_when: false + when: + - sestatus.stdout != 'Disabled' + - ansible_os_family == 'RedHat' + +- name: switching from non-containerized to containerized ceph mon + + vars: + mon_group_name: mons + + hosts: + - "{{ mon_group_name }}" + + serial: 1 + become: true + + tasks: + - include_vars: ../roles/ceph-common/defaults/main.yml + - include_vars: ../roles/ceph-mon/defaults/main.yml + - include_vars: ../roles/ceph-restapi/defaults/main.yml + - include_vars: ../group_vars/all + failed_when: false + - include_vars: ../group_vars/mons + failed_when: false + - include_vars: ../group_vars/restapis + failed_when: false + + - name: select a running monitor + set_fact: mon_host={{ item }} + with_items: groups.mons + when: item != inventory_hostname + + - name: get current ceph fsid + command: ceph fsid + register: ceph_fsid + changed_when: false + delegate_to: "{{ mon_host }}" + + - name: stop ceph mon bare metal service + service: + name: "ceph-mon@{{ ansible_hostname }}" + state: stopped + enabled: no + + # NOTE(leseb): should we also create systemd files + # intead of running raw docker commands? + # It is probably more elegant but will require a template file... + # which would make this single file playbook more complex + + - set_fact: + ceph_uid: 64045 + when: ceph_mon_docker_image_tag | match("latest") + + - set_fact: + ceph_uid: 64045 + when: ceph_mon_docker_image_tag | search("ubuntu") + + - set_fact: + ceph_uid: 167 + when: ceph_mon_docker_image_tag | search("centos") + + - set_fact: + ceph_uid: 167 + when: ceph_mon_docker_image_tag | search("fedora") + + - name: set proper ownership on ceph directories + file: + path: "{{ item }}" + owner: "{{ ceph_uid }}" + recurse: yes + with_items: + - /var/lib/ceph + - /etc/ceph + + - name: start ceph mon container image + docker: + image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}:{{ ceph_mon_docker_image_tag }}" + name: "{{ ansible_hostname }}" + net: "host" + state: "running" + privileged: "{{ mon_docker_privileged }}" + env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},CEPH_FSID={{ ceph_fsid.stdout }},{{ ceph_mon_extra_envs }}" + volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro" + + - name: waiting for the monitor to join the quorum... + shell: | + ceph -s --cluster {{ cluster }} | grep monmap | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }} + register: result + until: result.rc == 0 + retries: 5 + delay: 10 + changed_when: false + delegate_to: "{{ mon_host }}" + + +- name: switching from non-containerized to containerized ceph osd + + vars: + osd_group_name: osds + + hosts: + - "{{ osd_group_name }}" + + serial: 1 + become: true + + tasks: + - include_vars: ../roles/ceph-common/defaults/main.yml + - include_vars: ../roles/ceph-osd/defaults/main.yml + - include_vars: ../group_vars/all + failed_when: false + - include_vars: ../group_vars/osds + failed_when: false + + - name: collect osd ids + shell: | + if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | cut -d '-' -f 2 ; fi + register: osd_ids + changed_when: false + + - name: collect osd devices + shell: | + blkid | awk '/ceph data/ { sub ("1:", "", $1); print $1 }' + register: ceph_osd_docker_devices + changed_when: false + + - name: stop ceph osd service + service: + name: "ceph-osd@{{ item }}" + state: stopped + enabled: no + with_items: "{{ osd_ids.stdout_lines }}" + + - set_fact: + ceph_uid: 64045 + when: ceph_osd_docker_image_tag | match("latest") + + - set_fact: + ceph_uid: 64045 + when: ceph_osd_docker_image_tag | search("ubuntu") + + - set_fact: + ceph_uid: 167 + when: ceph_osd_docker_image_tag | search("centos") + + - set_fact: + ceph_uid: 167 + when: ceph_osd_docker_image_tag | search("fedora") + + - name: set proper ownership on ceph directories + file: + path: "{{ item }}" + owner: "{{ ceph_uid }}" + recurse: yes + with_items: + - /var/lib/ceph + - /etc/ceph + + - name: check if containerized osds are already running + shell: | + docker ps | grep -sq {{ item | regex_replace('/', '') }} + changed_when: false + with_items: "{{ ceph_osd_docker_devices.stdout_lines }}" + register: osd_running + + - name: unmount all the osd directories + mount: + name: "/var/lib/ceph/osd/{{ cluster }}-{{ item.0 }}" + state: unmounted + src: "{{ item.1 }}" + fstype: xfs + with_together: + - "{{ osd_ids.stdout_lines }}" + - "{{ ceph_osd_docker_devices.stdout_lines }}" + - "{{ osd_running.results }}" + when: item.2.rc != 0 + + - name: start ceph osd container image(s) + docker: + image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}" + name: "{{ ansible_hostname }}-osd-{{ item | regex_replace('/', '') }}" + net: host + pid: host + state: started + privileged: yes + env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_extra_env }}" + volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro,/dev:/dev,/run:/run" + with_items: "{{ ceph_osd_docker_devices.stdout_lines }}" + + - name: waiting for clean pgs... + shell: | + test "$(ceph pg stat --cluster {{ cluster }} | sed 's/^.*pgs://;s/active+clean.*//;s/ //')" -eq "$(ceph pg stat --cluster {{ cluster }} | sed 's/pgs.*//;s/^.*://;s/ //')" && ceph health --cluster {{ cluster }} | egrep -sq "HEALTH_OK|HEALTH_WARN" + register: result + until: result.rc == 0 + retries: 10 + delay: 10 + changed_when: false + delegate_to: "{{ mon_host }}" + + +- name: switching from non-containerized to containerized ceph mds + + vars: + mds_group_name: mdss + + hosts: + - "{{ mds_group_name }}" + + serial: 1 + become: true + + tasks: + - include_vars: ../roles/ceph-common/defaults/main.yml + - include_vars: ../roles/ceph-mds/defaults/main.yml + - include_vars: ../group_vars/all + failed_when: false + - include_vars: ../group_vars/mdss + failed_when: false + + - name: stop ceph mds service + service: + name: "ceph-mds@{{ ansible_hostname }}" + state: stopped + enabled: no + + - set_fact: + ceph_uid: 64045 + when: ceph_mds_docker_image_tag | match("latest") + + - set_fact: + ceph_uid: 64045 + when: ceph_mds_docker_image_tag | search("ubuntu") + + - set_fact: + ceph_uid: 167 + when: ceph_mds_docker_image_tag | search("centos") + + - set_fact: + ceph_uid: 167 + when: ceph_mds_docker_image_tag | search("fedora") + + - name: set proper ownership on ceph directories + file: + path: "{{ item }}" + owner: "{{ ceph_uid }}" + recurse: yes + with_items: + - /var/lib/ceph + - /etc/ceph + + - name: start ceph metadata container image + docker: + image: "{{ ceph_mds_docker_username }}/{{ ceph_mds_docker_imagename }}:{{ ceph_mds_docker_image_tag }}" + name: ceph-{{ ansible_hostname }}-mds + net: host + state: running + env: "CEPH_DAEMON=MDS,CEPHFS_CREATE=1,{{ ceph_mds_docker_extra_env }}" + volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro" + + +- name: switching from non-containerized to containerized ceph rgw + + vars: + rgw_group_name: rgws + + hosts: + - "{{ rgw_group_name }}" + + serial: 1 + become: true + + tasks: + - include_vars: ../roles/ceph-common/defaults/main.yml + - include_vars: ../roles/ceph-rgw/defaults/main.yml + - include_vars: ../group_vars/all + failed_when: false + - include_vars: ../group_vars/rgws + failed_when: false + + - name: stop ceph rgw service + service: + name: "ceph-rgw@{{ ansible_hostname }}" + state: stopped + enabled: no + + - set_fact: + ceph_uid: 64045 + when: ceph_rgw_docker_image_tag | match("latest") + + - set_fact: + ceph_uid: 64045 + when: ceph_rgw_docker_image_tag | search("ubuntu") + + - set_fact: + ceph_uid: 167 + when: ceph_rgw_docker_image_tag | search("centos") + + - set_fact: + ceph_uid: 167 + when: ceph_rgw_docker_image_tag | search("fedora") + + - name: set proper ownership on ceph directories + file: + path: "{{ item }}" + owner: "{{ ceph_uid }}" + recurse: yes + with_items: + - /var/lib/ceph + - /etc/ceph + + - name: start ceph rados gateway container image + docker: + image: "{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}:{{ ceph_rgw_docker_image_tag }}" + name: ceph-{{ ansible_hostname }}-rgw + expose: "{{ ceph_rgw_civetweb_port }}" + ports: "{{ ceph_rgw_civetweb_port }}:{{ ceph_rgw_civetweb_port }}" + state: running + env: "CEPH_DAEMON=RGW,{{ ceph_rgw_docker_extra_env }}" + volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro" + + +- name: switching from non-containerized to containerized ceph rbd-mirror + + vars: + rbdmirror_group_name: rbd_mirrors + + hosts: + - "{{ rbdmirror_group_name }}" + + serial: 1 + become: true + + tasks: + - include_vars: ../roles/ceph-common/defaults/main.yml + - include_vars: ../roles/ceph-rbd-mirror/defaults/main.yml + - include_vars: ../group_vars/all + failed_when: false + - include_vars: ../group_vars/rbd-mirrors + failed_when: false + + - name: stop ceph rbd mirror service + service: + name: "ceph-rbd-mirror@{{ ansible_hostname }}" + state: stopped + enabled: no + + - set_fact: + ceph_uid: 64045 + when: ceph_rbd_mirror_docker_image_tag | match("latest") + + - set_fact: + ceph_uid: 64045 + when: ceph_rbd_mirror_docker_image_tag | search("ubuntu") + + - set_fact: + ceph_uid: 167 + when: ceph_rbd_mirror_docker_image_tag | search("centos") + + - set_fact: + ceph_uid: 167 + when: ceph_rbd_mirror_docker_image_tag | search("fedora") + + - name: set proper ownership on ceph directories + file: + path: "{{ item }}" + owner: "{{ ceph_uid }}" + recurse: yes + with_items: + - /var/lib/ceph + - /etc/ceph + + - name: start ceph rbd mirror container image + docker: + image: "{{ ceph_rbd_mirror_docker_username }}/{{ ceph_rbd_mirror_docker_imagename }}:{{ ceph_rbd_mirror_docker_image_tag }}" + name: "{{ ansible_hostname }}" + net: host + state: running + volumes: "/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro" + + +- name: switching from non-containerized to containerized ceph nfs + + vars: + nfs_group_name: nfss + + hosts: + - "{{ nfs_group_name }}" + + serial: 1 + become: true + + tasks: + - include_vars: ../roles/ceph-common/defaults/main.yml + - include_vars: ../roles/ceph-nfs/defaults/main.yml + - include_vars: ../group_vars/all + failed_when: false + - include_vars: ../group_vars/nfss + failed_when: false + + - name: stop ceph nfs service + service: + name: "ceph-nfs@{{ ansible_hostname }}" + state: stopped + enabled: no + + - set_fact: + ceph_uid: 64045 + when: ceph_nfs_docker_image_tag | match("latest") + + - set_fact: + ceph_uid: 64045 + when: ceph_nfs_docker_image_tag | search("ubuntu") + + - set_fact: + ceph_uid: 167 + when: ceph_nfs_docker_image_tag | search("centos") + + - set_fact: + ceph_uid: 167 + when: ceph_nfs_docker_image_tag | search("fedora") + + - name: set proper ownership on ceph directories + file: + path: "{{ item }}" + owner: "{{ ceph_uid }}" + recurse: yes + with_items: + - /var/lib/ceph + - /etc/ceph + + - name: start ceph nfs container image + docker: + image: "{{ ceph_nfs_docker_username }}/{{ ceph_nfs_docker_imagename }}:{{ ceph_nfs_docker_image_tag }}" + name: "{{ ansible_hostname }}" + net: "host" + state: "running" + privileged: true + ports: "{{ ceph_nfs_port }}:{{ ceph_nfs_port }},111:111" + env: "CEPH_DAEMON=NFS,CEPH_PUBLIC_NETWORK={{ ceph_nfs_docker_subnet }},{{ ceph_nfs_extra_envs }}" + volumes: "/etc/ceph:/etc/ceph,/etc/ganesha:/etc/ganesha,/etc/localtime:/etc/localtime:ro"