ceph-ansible/infrastructure-playbooks/rolling_update.yml

---
# This playbook does a rolling update for all the Ceph services
#
# The value of 'serial:' adjusts the number of servers to be updated simultaneously.
# We recommend a value of 1, which means hosts of a group (e.g: monitor) will be
# upgraded one by one. It is really crucial for the update process to happen
# in a serialized fashion. DO NOT CHANGE THIS VALUE.
#
# The four roles that apply to the ceph hosts will be applied: ceph-common,
# ceph-mon, ceph-osd and ceph-mds. So any changes to configuration, package updates, etc,
# will be applied as part of the rolling update process.
#

# /!\ DO NOT FORGET TO CHANGE THE RELEASE VERSION FIRST! /!\

- name: confirm whether user really meant to upgrade the cluster
  hosts: localhost

  vars_prompt:
    - name: ireallymeanit
      prompt: Are you sure you want to upgrade the cluster?
      default: 'no'
      private: no

  tasks:
  - name: exit playbook, if user did not mean to upgrade cluster
    fail:
      msg: >
        "Exiting rolling_update.yml playbook, cluster was NOT upgraded.
         To upgrade the cluster, either say 'yes' on the prompt or
         use `-e ireallymeanit=yes` on the command line when
         invoking the playbook"
    when: ireallymeanit != 'yes'


- name: gather facts and check the init system

  hosts:
    - "{{ mon_group_name|default('mons') }}"
    - "{{ osd_group_name|default('osds') }}"
    - "{{ mds_group_name|default('mdss') }}"
    - "{{ rgw_group_name|default('rgws') }}"
    - "{{ mgr_group_name|default('mgrs') }}"

  become: True
  tasks:
    - debug: msg="gather facts on all Ceph hosts for following reference"

    - set_fact: rolling_update=true

- name: upgrade ceph mon cluster

  vars:
    health_mon_check_retries: 5
    health_mon_check_delay: 10
    upgrade_ceph_packages: True

  hosts:
    - "{{ mon_group_name|default('mons') }}"

  serial: 1
  become: True

  pre_tasks:
    - name: set mon_host_count
      set_fact: mon_host_count={{ groups[mon_group_name] | length }}

    - debug: msg="WARNING - upgrading a ceph cluster with only one monitor node ({{ inventory_hostname }})"
      when: mon_host_count | int == 1

    - name: stop ceph mons with upstart
      service:
        name: ceph-mon
        state: stopped
        args: id={{ ansible_hostname }}
      when: ansible_service_mgr == 'upstart'

    - name: stop ceph mons with sysvinit
      service:
        name: ceph
        state: stopped
      when: ansible_service_mgr == 'sysvinit'

    - name: stop ceph mons with systemd
      service:
        name: ceph-mon@{{ ansible_hostname }}
        state: stopped
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not mon_containerized_deployment

  roles:
    - ceph-mon

  post_tasks:
    - name: start ceph mons with upstart
      service:
        name: ceph-mon
        state: started
        args: id={{ ansible_hostname }}
      when: ansible_service_mgr == 'upstart'

    - name: start ceph mons with sysvinit
      service:
        name: ceph
        state: started
      when: ansible_service_mgr == 'sysvinit'

    - name: start ceph mons with systemd
      service:
        name: ceph-mon@{{ ansible_hostname }}
        state: started
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not mon_containerized_deployment

    - name: restart containerized ceph mons with systemd
      service:
        name: ceph-mon@{{ ansible_hostname }}
        state: restarted
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - mon_containerized_deployment

    - name: set mon_host_count
      set_fact: mon_host_count={{ groups[mon_group_name] | length }}

    - name: select a running monitor if multiple monitors
      set_fact: mon_host={{ item }}
      with_items: "{{ groups[mon_group_name] }}"
      when:
        - mon_host_count | int > 1
        - item != inventory_hostname

    - name: select first monitor if only one monitor
      set_fact: mon_host={{ item }}
      with_items: "{{ groups[mon_group_name][0] }}"
      when:
        - mon_host_count | int == 1

    - name: waiting for the monitor to join the quorum...
      shell: |
        ceph -s --cluster {{ cluster }} | grep election | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }}
      register: result
      until: result.rc == 0
      retries: "{{ health_mon_check_retries }}"
      delay: "{{ health_mon_check_delay }}"
      delegate_to: "{{ mon_host }}"
      when: not mon_containerized_deployment

    - name: waiting for the containerized monitor to join the quorum...
      shell: |
        docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph -s --cluster {{ cluster }} | grep quorum | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }}
      register: result
      until: result.rc == 0
      retries: "{{ health_mon_check_retries }}"
      delay: "{{ health_mon_check_delay }}"
      delegate_to: "{{ mon_host }}"
      when: mon_containerized_deployment


- name: upgrade ceph osds cluster

  vars:
    health_osd_check_retries: 40
    health_osd_check_delay: 30
    upgrade_ceph_packages: True

  hosts:
    - "{{ osd_group_name|default('osds') }}"

  serial: 1
  become: True

  pre_tasks:
    - name: set osd flags
      command: ceph osd set {{ item }} --cluster {{ cluster }}
      with_items:
        - noout
        - noscrub
        - nodeep-scrub
      delegate_to: "{{ groups[mon_group_name][0] }}"
      when: not mon_containerized_deployment

    - name: set containerized osd flags
      command: |
          docker exec ceph-osd-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph osd set {{ item }} --cluster {{ cluster }}
      with_items:
        - noout
        - noscrub
        - nodeep-scrub
      delegate_to: "{{ groups[mon_group_name][0] }}"
      when: mon_containerized_deployment

    - name: get osd numbers
      shell: "if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi"
      register: osd_ids
      changed_when: false
      when: not osd_containerized_deployment

    - name: stop ceph osds with upstart
      service:
        name: ceph-osd-all
        state: stopped
      when: ansible_service_mgr == 'upstart'

    - name: stop ceph osds with sysvinit
      service:
        name: ceph
        state: stopped
      when: ansible_service_mgr == 'sysvinit'

    - name: stop ceph osds with systemd
      service:
        name: ceph-osd@{{item}}
        state: stopped
        enabled: yes
      with_items: "{{ osd_ids.stdout_lines }}"
      when:
        - ansible_service_mgr == 'systemd'
        - not osd_containerized_deployment

  roles:
    - ceph-osd

  post_tasks:
    - name: get osd numbers
      shell: "if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi"
      register: osd_ids
      changed_when: false
      when: not osd_containerized_deployment

    - name: start ceph osds with upstart
      service:
        name: ceph-osd-all
        state: started
      when: ansible_service_mgr == 'upstart'

    - name: start ceph osds with sysvinit
      service:
        name: ceph
        state: started
      when: ansible_service_mgr == 'sysvinit'

    - name: start ceph osds with systemd
      service:
        name: ceph-osd@{{item}}
        state: started
        enabled: yes
      with_items: "{{ osd_ids.stdout_lines }}"
      when:
        - ansible_service_mgr == 'systemd'
        - not osd_containerized_deployment

    - name: restart containerized ceph osds with systemd
      service:
        name: ceph-osd@{{ item | basename }}
        state: restarted
        enabled: yes
      with_items: "{{ ceph_osd_docker_devices }}"
      when:
        - ansible_service_mgr == 'systemd'
        - osd_containerized_deployment

    - name: waiting for clean pgs...
      shell: |
        test "$(ceph pg stat --cluster {{ cluster }} | tr , '\n'  | sed 's/^.*pgs: //' | awk '/active.clean/ { SUM += $1 } END {print SUM}')" -eq "$(ceph pg stat --cluster {{ cluster }}  | sed 's/pgs.*//;s/^.*://;s/ //')" && ceph health --cluster {{ cluster }}  | egrep -sq "HEALTH_OK|HEALTH_WARN"
      register: result
      until: result.rc == 0
      retries: "{{ health_osd_check_retries }}"
      delay: "{{ health_osd_check_delay }}"
      delegate_to: "{{ groups[mon_group_name][0] }}"
      when: not osd_containerized_deployment

    - name: container - waiting for clean pgs...
      shell: |
        test "$(docker exec ceph-osd-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph pg stat --cluster {{ cluster }} | sed 's/^.*pgs://;s/active+clean.*//;s/ //')" -eq "$(docker exec {{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph pg stat --cluster {{ cluster }} | sed 's/pgs.*//;s/^.*://;s/ //')" && docker exec {{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph health --cluster {{ cluster }}  | egrep -sq "HEALTH_OK|HEALTH_WARN"
      register: result
      until: result.rc == 0
      retries: "{{ health_osd_check_retries }}"
      delay: "{{ health_osd_check_delay }}"
      delegate_to: "{{ groups[mon_group_name][0] }}"
      when: osd_containerized_deployment

    - name: unset osd flags
      command: ceph osd unset {{ item }} --cluster {{ cluster }}
      with_items:
        - noout
        - noscrub
        - nodeep-scrub
      delegate_to: "{{ groups[mon_group_name][0] }}"
      when: not osd_containerized_deployment

    - name: unset containerized osd flags
      command: |
          docker exec ceph-osd-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph osd unset {{ item }} --cluster {{ cluster }}
      with_items:
        - noout
        - noscrub
        - nodeep-scrub
      delegate_to: "{{ groups[mon_group_name][0] }}"
      when: osd_containerized_deployment


- name: upgrade ceph mdss cluster

  vars:
    upgrade_ceph_packages: True

  hosts:
    - "{{ mds_group_name|default('mdss') }}"

  serial: 1
  become: True

  pre_tasks:
    - name: stop ceph mdss with upstart
      service:
        name: ceph-mds
        state: stopped
        args: id={{ ansible_hostname }}
      when: ansible_service_mgr == 'upstart'

    - name: stop ceph mdss with sysvinit
      service:
        name: ceph
        state: stopped
        args: mds
      when: ansible_service_mgr == 'sysvinit'

    - name: stop ceph mdss with systemd
      service:
        name: ceph-mds@{{ ansible_hostname }}
        state: stopped
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not mds_containerized_deployment

  roles:
    - ceph-mds

  post_tasks:
    - name: start ceph mdss with upstart
      service:
        name: ceph-mds
        state: started
        args: id={{ ansible_hostname }}
      when: ansible_service_mgr == 'upstart'

    - name: start ceph mdss with sysvinit
      service:
        name: ceph
        state: started
        args: mds
      when: ansible_service_mgr == 'sysvinit'

    - name: start ceph mdss with systemd
      service:
        name: ceph-mds@{{ ansible_hostname }}
        state: started
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not mds_containerized_deployment

    - name: restart ceph mdss
      service:
        name: ceph-mds@{{ ansible_hostname }}
        state: restarted
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - mds_containerized_deployment


- name: upgrade ceph rgws cluster

  vars:
    upgrade_ceph_packages: True

  hosts:
    - "{{ rgw_group_name|default('rgws') }}"

  serial: 1
  become: True

  pre_tasks:
    - name: stop ceph rgws with upstart
      service:
        name: ceph-radosgw
        state: stopped
      when: ansible_service_mgr == 'upstart'

    - name: stop ceph rgws with sysvinit
      service:
        name: radosgw
        state: stopped
      when: ansible_service_mgr == 'sysvinit'

    - name: stop ceph rgws with systemd
      service:
        name: ceph-radosgw@rgw.{{ ansible_hostname }}
        state: stopped
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not rgw_containerized_deployment

  roles:
    - ceph-rgw

  post_tasks:
    - name: start ceph rgws with upstart
      service:
        name: ceph-radosgw
        state: started
      when: ansible_service_mgr == 'upstart'

    - name: start ceph rgws with sysvinit
      service:
        name: radosgw
        state: started
      when: ansible_service_mgr == 'sysvinit'

    - name: start ceph rgws with systemd
      service:
        name: ceph-radosgw@rgw.{{ ansible_hostname }}
        state: started
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not rgw_containerized_deployment

    - name: restart containerized ceph rgws with systemd
      service:
        name: ceph-rgw@{{ ansible_hostname }}
        state: restarted
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - rgw_containerized_deployment