ceph-ansible/infrastructure-playbooks/rolling_update.yml

---
# This playbook does a rolling update for all the Ceph services
#
# The value of 'serial:' adjusts the number of servers to be updated simultaneously.
# We recommend a value of 1, which means hosts of a group (e.g: monitor) will be
# upgraded one by one. It is really crucial for the update process to happen
# in a serialized fashion. DO NOT CHANGE THIS VALUE.
#
#
# If you run a Ceph community version, you have to change the variable: ceph_stable_release to the new release
#
# If you run Red Hat Ceph Storage and are doing a **major** update (e.g: from 2 to 3), you have two options:
#   - if you use a CDN, you have to change the ceph_rhcs_version to a newer one
#   - if you use an ISO, you have to change the ceph_rhcs_iso_path to the directory containing the new Ceph version
#

- name: confirm whether user really meant to upgrade the cluster
  hosts: localhost

  vars_prompt:
    - name: ireallymeanit
      prompt: Are you sure you want to upgrade the cluster?
      default: 'no'
      private: no

  tasks:
  - name: exit playbook, if user did not mean to upgrade cluster
    fail:
      msg: >
        "Exiting rolling_update.yml playbook, cluster was NOT upgraded.
         To upgrade the cluster, either say 'yes' on the prompt or
         use `-e ireallymeanit=yes` on the command line when
         invoking the playbook"        
    when: ireallymeanit != 'yes'


- name: gather facts and check the init system

  hosts:
    - "{{ mon_group_name|default('mons') }}"
    - "{{ osd_group_name|default('osds') }}"
    - "{{ mds_group_name|default('mdss') }}"
    - "{{ rgw_group_name|default('rgws') }}"
    - "{{ mgr_group_name|default('mgrs') }}"
    - "{{ client_group_name|default('clients') }}"

  become: True
  tasks:
    - debug: msg="gather facts on all Ceph hosts for following reference"

    - set_fact: rolling_update=true

- name: upgrade ceph mon cluster

  vars:
    health_mon_check_retries: 5
    health_mon_check_delay: 15
    upgrade_ceph_packages: True

  hosts:
    - "{{ mon_group_name|default('mons') }}"

  serial: 1
  become: True

  pre_tasks:
    - name: set mon_host_count
      set_fact: mon_host_count={{ groups[mon_group_name] | length }}

    - debug: msg="WARNING - upgrading a ceph cluster with only one monitor node ({{ inventory_hostname }})"
      when: mon_host_count | int == 1

    - name: stop ceph mons with upstart
      service:
        name: ceph-mon
        state: stopped
        args: id={{ ansible_hostname }}
      when: ansible_service_mgr == 'upstart'

    - name: stop ceph mons with sysvinit
      service:
        name: ceph
        state: stopped
      when: ansible_service_mgr == 'sysvinit'

    - name: stop ceph mons with systemd
      service:
        name: ceph-mon@{{ ansible_hostname }}
        state: stopped
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not containerized_deployment

  roles:
    - ceph-defaults
    - ceph-config
    - { role: ceph-common, when: not containerized_deployment }
    - { role: ceph-docker-common, when: containerized_deployment }
    - ceph-mon

  post_tasks:
    - name: start ceph mons with upstart
      service:
        name: ceph-mon
        state: started
        args: id={{ ansible_hostname }}
      when: ansible_service_mgr == 'upstart'

    - name: start ceph mons with sysvinit
      service:
        name: ceph
        state: started
      when: ansible_service_mgr == 'sysvinit'

    - name: start ceph mons with systemd
      service:
        name: ceph-mon@{{ ansible_hostname }}
        state: started
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not containerized_deployment

    - name: restart containerized ceph mons with systemd
      service:
        name: ceph-mon@{{ ansible_hostname }}
        state: restarted
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - containerized_deployment

    - name: set mon_host_count
      set_fact: mon_host_count={{ groups[mon_group_name] | length }}

    - name: select a running monitor if multiple monitors
      set_fact: mon_host={{ item }}
      with_items: "{{ groups[mon_group_name] }}"
      when:
        - mon_host_count | int > 1
        - item != inventory_hostname

    - name: select first monitor if only one monitor
      set_fact: mon_host={{ item }}
      with_items: "{{ groups[mon_group_name][0] }}"
      when:
        - mon_host_count | int == 1

    - name: non container | waiting for the monitor to join the quorum...
      command: ceph --cluster "{{ cluster }}" -s --format json
      register: ceph_health_raw
      until: >
        hostvars[mon_host]['ansible_hostname'] in (ceph_health_raw.stdout | from_json)["quorum_names"]        
      retries: "{{ health_mon_check_retries }}"
      delay: "{{ health_mon_check_delay }}"
      delegate_to: "{{ mon_host }}"
      when:
        - not containerized_deployment

    - name: container | waiting for the containerized monitor to join the quorum...
      command: docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster "{{ cluster }}" -s --format json
      register: ceph_health_raw
      until: >
        hostvars[mon_host]['ansible_hostname'] in (ceph_health_raw.stdout | from_json)["quorum_names"]        
      retries: "{{ health_mon_check_retries }}"
      delay: "{{ health_mon_check_delay }}"
      delegate_to: "{{ mon_host }}"
      when:
        - containerized_deployment

    - name: set osd flags
      command: ceph osd set {{ item }} --cluster {{ cluster }}
      with_items:
        - noout
        - noscrub
        - nodeep-scrub
      delegate_to: "{{ groups[mon_group_name][0] }}"
      when: not containerized_deployment

    - name: set containerized osd flags
      command: |
          docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph osd set {{ item }} --cluster {{ cluster }}          
      with_items:
        - noout
        - noscrub
        - nodeep-scrub
      delegate_to: "{{ groups[mon_group_name][0] }}"
      when: containerized_deployment


- name: upgrade ceph osds cluster

  vars:
    health_osd_check_retries: 40
    health_osd_check_delay: 30
    upgrade_ceph_packages: True

  hosts:
    - "{{ osd_group_name|default('osds') }}"

  serial: 1
  become: True

  pre_tasks:
    - name: get osd numbers
      shell: "if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi"
      register: osd_ids
      changed_when: false
      when: not containerized_deployment

    - name: stop ceph osds with upstart
      service:
        name: ceph-osd-all
        state: stopped
      when: ansible_service_mgr == 'upstart'

    - name: stop ceph osds with sysvinit
      service:
        name: ceph
        state: stopped
      when: ansible_service_mgr == 'sysvinit'

    - name: stop ceph osds with systemd
      service:
        name: ceph-osd@{{item}}
        state: stopped
        enabled: yes
      with_items: "{{ osd_ids.stdout_lines }}"
      when:
        - ansible_service_mgr == 'systemd'
        - not containerized_deployment

  roles:
    - ceph-defaults
    - ceph-config
    - { role: ceph-common, when: not containerized_deployment }
    - { role: ceph-docker-common, when: containerized_deployment }
    - ceph-osd

  post_tasks:
    - name: get osd numbers
      shell: "if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi"
      register: osd_ids
      changed_when: false
      when: not containerized_deployment

    - name: start ceph osds with upstart
      service:
        name: ceph-osd-all
        state: started
      when: ansible_service_mgr == 'upstart'

    - name: start ceph osds with sysvinit
      service:
        name: ceph
        state: started
      when: ansible_service_mgr == 'sysvinit'

    - name: start ceph osds with systemd
      service:
        name: ceph-osd@{{item}}
        state: started
        enabled: yes
      with_items: "{{ osd_ids.stdout_lines }}"
      when:
        - ansible_service_mgr == 'systemd'
        - not containerized_deployment

    - name: restart containerized ceph osds with systemd
      service:
        name: ceph-osd@{{ item | basename }}
        state: restarted
        enabled: yes
      with_items: "{{ devices }}"
      when:
        - ansible_service_mgr == 'systemd'
        - containerized_deployment

    - name: set_fact docker_exec_cmd_osd
      set_fact:
        docker_exec_cmd_update_osd: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
      when:
        - containerized_deployment

    - name: get num_pgs - non container
      command: "{{ docker_exec_cmd_update_osd|default('') }} ceph --cluster {{ cluster }} -s --format json"
      register: ceph_pgs
      delegate_to: "{{ groups[mon_group_name][0] }}"

    - name: waiting for clean pgs...
      command: "{{ docker_exec_cmd_update_osd|default('') }} ceph --cluster {{ cluster }} -s --format json"
      register: ceph_health_post
      until: >
        ((ceph_health_post.stdout | from_json).pgmap.pgs_by_state | length) == 1
        and
        (ceph_health_post.stdout | from_json).pgmap.pgs_by_state.0.state_name == "active+clean"        
      delegate_to: "{{ groups[mon_group_name][0] }}"
      retries: "{{ health_osd_check_retries }}"
      delay: "{{ health_osd_check_delay }}"
      when:
        - (ceph_pgs.stdout | from_json).pgmap.num_pgs != 0


- name: unset osd flags

  hosts:
    - "{{ mon_group_name|default('mons') }}"

  become: True

  roles:
    - ceph-defaults

  tasks:
    - name: set_fact docker_exec_cmd_osd
      set_fact:
        docker_exec_cmd_update_osd: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
      when:
        - containerized_deployment

    - name: unset osd flags
      command: "{{ docker_exec_cmd_update_osd|default('') }} ceph osd unset {{ item }} --cluster {{ cluster }}"
      with_items:
        - noout
        - noscrub
        - nodeep-scrub
      delegate_to: "{{ groups[mon_group_name][0] }}"

    - name: get osd versions
      command: "{{ docker_exec_cmd_update_osd|default('') }} ceph --cluster {{ cluster }} versions"
      register: ceph_versions
      delegate_to: "{{ groups[mon_group_name][0] }}"

    - name: set_fact ceph_versions_osd
      set_fact:
        ceph_versions_osd: "{{ (ceph_versions.stdout|from_json).osd }}"
      delegate_to: "{{ groups[mon_group_name][0] }}"

    # length == 1 means there is a single osds versions entry
    # thus all the osds are running the same version
    - name: complete osds upgrade
      command: "{{ docker_exec_cmd_update_osd|default('') }} ceph --cluster {{ cluster }} osd require-osd-release luminous"
      delegate_to: "{{ groups[mon_group_name][0] }}"
      when:
        - (ceph_versions.stdout|from_json).osd | length == 1
        - ceph_versions_osd | string | search("ceph version 12")


- name: upgrade ceph mdss cluster

  vars:
    upgrade_ceph_packages: True

  hosts:
    - "{{ mds_group_name|default('mdss') }}"

  serial: 1
  become: True

  pre_tasks:
    - name: stop ceph mdss with upstart
      service:
        name: ceph-mds
        state: stopped
        args: id={{ ansible_hostname }}
      when: ansible_service_mgr == 'upstart'

    - name: stop ceph mdss with sysvinit
      service:
        name: ceph
        state: stopped
        args: mds
      when: ansible_service_mgr == 'sysvinit'

    - name: stop ceph mdss with systemd
      service:
        name: ceph-mds@{{ ansible_hostname }}
        state: stopped
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not containerized_deployment

  roles:
    - ceph-defaults
    - ceph-config
    - { role: ceph-common, when: not containerized_deployment }
    - { role: ceph-docker-common, when: containerized_deployment }
    - ceph-mds

  post_tasks:
    - name: start ceph mdss with upstart
      service:
        name: ceph-mds
        state: started
        args: id={{ ansible_hostname }}
      when: ansible_service_mgr == 'upstart'

    - name: start ceph mdss with sysvinit
      service:
        name: ceph
        state: started
        args: mds
      when: ansible_service_mgr == 'sysvinit'

    - name: start ceph mdss with systemd
      service:
        name: ceph-mds@{{ ansible_hostname }}
        state: started
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not containerized_deployment

    - name: restart ceph mdss
      service:
        name: ceph-mds@{{ ansible_hostname }}
        state: restarted
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - containerized_deployment


- name: upgrade ceph rgws cluster

  vars:
    upgrade_ceph_packages: True

  hosts:
    - "{{ rgw_group_name|default('rgws') }}"

  serial: 1
  become: True

  pre_tasks:
    - name: stop ceph rgws with upstart
      service:
        name: ceph-radosgw
        state: stopped
      when: ansible_service_mgr == 'upstart'

    - name: stop ceph rgws with sysvinit
      service:
        name: radosgw
        state: stopped
      when: ansible_service_mgr == 'sysvinit'

    - name: stop ceph rgws with systemd
      service:
        name: ceph-radosgw@rgw.{{ ansible_hostname }}
        state: stopped
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not containerized_deployment

  roles:
    - ceph-defaults
    - ceph-config
    - { role: ceph-common, when: not containerized_deployment }
    - { role: ceph-docker-common, when: containerized_deployment }
    - ceph-rgw

  post_tasks:
    - name: start ceph rgws with upstart
      service:
        name: ceph-radosgw
        state: started
      when: ansible_service_mgr == 'upstart'

    - name: start ceph rgws with sysvinit
      service:
        name: radosgw
        state: started
      when: ansible_service_mgr == 'sysvinit'

    - name: start ceph rgws with systemd
      service:
        name: ceph-radosgw@rgw.{{ ansible_hostname }}
        state: started
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - not containerized_deployment

    - name: restart containerized ceph rgws with systemd
      service:
        name: ceph-radosgw@rgw.{{ ansible_hostname }}
        state: restarted
        enabled: yes
      when:
        - ansible_service_mgr == 'systemd'
        - containerized_deployment


- name: upgrade ceph client node

  vars:
    upgrade_ceph_packages: True

  hosts:
    - "{{ client_group_name|default('clients') }}"

  serial: 1
  become: True

  roles:
    - ceph-defaults
    - ceph-config
    - { role: ceph-common, when: not containerized_deployment }
    - { role: ceph-docker-common, when: containerized_deployment }
    - ceph-client


- name: upgrade ceph mgr node

  vars:
    upgrade_ceph_packages: True

  hosts:
    - "{{ mgr_group_name|default('mgrs') }}"

  serial: 1
  become: True

  pre_tasks:
    # this task has a failed_when: false to handle the scenario where no mgr existed before the upgrade
    - name: stop ceph mgrs
      service:
        name: ceph-mgr@{{ ansible_hostname }}
        state: stopped
        enabled: yes
      failed_when: false
      when:
        - not containerized_deployment

  roles:
    - ceph-defaults
    - ceph-config
    - { role: ceph-common, when: not containerized_deployment }
    - { role: ceph-docker-common, when: containerized_deployment }
    - { role: ceph-mgr, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }

  post_tasks:
    - name: start ceph mgrs
      service:
        name: ceph-mgr@{{ ansible_hostname }}
        state: started
        enabled: yes
      when:
        - not containerized_deployment

    - name: restart containerized ceph mgrs
      service:
        name: ceph-mgr@{{ ansible_hostname }}
        state: restarted
        enabled: yes
      when:
        - containerized_deployment


- name: show ceph status

  hosts:
    - "{{ mon_group_name|default('mons') }}"

  become: True

  roles:
    - ceph-defaults

  tasks:
    - name: set_fact docker_exec_cmd_status
      set_fact:
        docker_exec_cmd_status: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
      when:
        - containerized_deployment

    - name: show ceph status
      command: "{{ docker_exec_cmd_status|default('') }} ceph --cluster {{ cluster }} -s"
      delegate_to: "{{ groups[mon_group_name][0] }}"