ceph-ansible/infrastructure-playbooks/rolling_update.yml

532 lines
15 KiB
YAML

---
# This playbook does a rolling update for all the Ceph services
#
# The value of 'serial:' adjusts the number of servers to be updated simultaneously.
# We recommend a value of 1, which means hosts of a group (e.g: monitor) will be
# upgraded one by one. It is really crucial for the update process to happen
# in a serialized fashion. DO NOT CHANGE THIS VALUE.
#
#
# If you run a Ceph community version, you have to change the variable: ceph_stable_release to the new release
#
# If you run Red Hat Ceph Storage and are doing a **major** update (e.g: from 2 to 3), you have two options:
# - if you use a CDN, you have to change the ceph_rhcs_version to a newer one
# - if you use an ISO, you have to change the ceph_rhcs_iso_path to the directory containing the new Ceph version
#
- name: confirm whether user really meant to upgrade the cluster
hosts: localhost
vars_prompt:
- name: ireallymeanit
prompt: Are you sure you want to upgrade the cluster?
default: 'no'
private: no
tasks:
- name: exit playbook, if user did not mean to upgrade cluster
fail:
msg: >
"Exiting rolling_update.yml playbook, cluster was NOT upgraded.
To upgrade the cluster, either say 'yes' on the prompt or
use `-e ireallymeanit=yes` on the command line when
invoking the playbook"
when: ireallymeanit != 'yes'
- name: gather facts and check the init system
hosts:
- "{{ mon_group_name|default('mons') }}"
- "{{ osd_group_name|default('osds') }}"
- "{{ mds_group_name|default('mdss') }}"
- "{{ rgw_group_name|default('rgws') }}"
- "{{ mgr_group_name|default('mgrs') }}"
- "{{ client_group_name|default('clients') }}"
become: True
tasks:
- debug: msg="gather facts on all Ceph hosts for following reference"
- set_fact: rolling_update=true
- name: upgrade ceph mon cluster
vars:
health_mon_check_retries: 5
health_mon_check_delay: 15
upgrade_ceph_packages: True
hosts:
- "{{ mon_group_name|default('mons') }}"
serial: 1
become: True
pre_tasks:
- name: set mon_host_count
set_fact: mon_host_count={{ groups[mon_group_name] | length }}
- debug: msg="WARNING - upgrading a ceph cluster with only one monitor node ({{ inventory_hostname }})"
when: mon_host_count | int == 1
- name: stop ceph mons with upstart
service:
name: ceph-mon
state: stopped
args: id={{ ansible_hostname }}
when: ansible_service_mgr == 'upstart'
- name: stop ceph mons with sysvinit
service:
name: ceph
state: stopped
when: ansible_service_mgr == 'sysvinit'
- name: stop ceph mons with systemd
service:
name: ceph-mon@{{ ansible_hostname }}
state: stopped
enabled: yes
when:
- ansible_service_mgr == 'systemd'
- not containerized_deployment
roles:
- ceph-defaults
- { role: ceph-common, when: not containerized_deployment }
- { role: ceph-docker-common, when: containerized_deployment }
- ceph-mon
post_tasks:
- name: start ceph mons with upstart
service:
name: ceph-mon
state: started
args: id={{ ansible_hostname }}
when: ansible_service_mgr == 'upstart'
- name: start ceph mons with sysvinit
service:
name: ceph
state: started
when: ansible_service_mgr == 'sysvinit'
- name: start ceph mons with systemd
service:
name: ceph-mon@{{ ansible_hostname }}
state: started
enabled: yes
when:
- ansible_service_mgr == 'systemd'
- not containerized_deployment
- name: restart containerized ceph mons with systemd
service:
name: ceph-mon@{{ ansible_hostname }}
state: restarted
enabled: yes
when:
- ansible_service_mgr == 'systemd'
- containerized_deployment
- name: set mon_host_count
set_fact: mon_host_count={{ groups[mon_group_name] | length }}
- name: select a running monitor if multiple monitors
set_fact: mon_host={{ item }}
with_items: "{{ groups[mon_group_name] }}"
when:
- mon_host_count | int > 1
- item != inventory_hostname
- name: select first monitor if only one monitor
set_fact: mon_host={{ item }}
with_items: "{{ groups[mon_group_name][0] }}"
when:
- mon_host_count | int == 1
- name: waiting for the monitor to join the quorum...
command: ceph --cluster "{{ cluster }}" -s --format json
register: ceph_health_raw
until: >
hostvars[mon_host]['ansible_hostname'] in (ceph_health_raw.stdout | from_json)["quorum_names"]
retries: "{{ health_mon_check_retries }}"
delay: "{{ health_mon_check_delay }}"
delegate_to: "{{ mon_host }}"
when:
- not containerized_deployment
- name: waiting for the containerized monitor to join the quorum...
command: docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster "{{ cluster }}" -s --format json
register: ceph_health_raw
until: >
hostvars[mon_host]['ansible_hostname'] in (ceph_health_raw.stdout | from_json)["quorum_names"]
retries: "{{ health_mon_check_retries }}"
delay: "{{ health_mon_check_delay }}"
delegate_to: "{{ mon_host }}"
when:
- containerized_deployment
- name: set osd flags
command: ceph osd set {{ item }} --cluster {{ cluster }}
with_items:
- noout
- noscrub
- nodeep-scrub
delegate_to: "{{ groups[mon_group_name][0] }}"
when: not containerized_deployment
- name: set containerized osd flags
command: |
docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph osd set {{ item }} --cluster {{ cluster }}
with_items:
- noout
- noscrub
- nodeep-scrub
delegate_to: "{{ groups[mon_group_name][0] }}"
when: containerized_deployment
- name: upgrade ceph osds cluster
vars:
health_osd_check_retries: 40
health_osd_check_delay: 30
upgrade_ceph_packages: True
hosts:
- "{{ osd_group_name|default('osds') }}"
serial: 1
become: True
pre_tasks:
- name: get osd numbers
shell: "if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi"
register: osd_ids
changed_when: false
when: not containerized_deployment
- name: stop ceph osds with upstart
service:
name: ceph-osd-all
state: stopped
when: ansible_service_mgr == 'upstart'
- name: stop ceph osds with sysvinit
service:
name: ceph
state: stopped
when: ansible_service_mgr == 'sysvinit'
- name: stop ceph osds with systemd
service:
name: ceph-osd@{{item}}
state: stopped
enabled: yes
with_items: "{{ osd_ids.stdout_lines }}"
when:
- ansible_service_mgr == 'systemd'
- not containerized_deployment
roles:
- ceph-defaults
- { role: ceph-common, when: not containerized_deployment }
- { role: ceph-docker-common, when: containerized_deployment }
- ceph-osd
post_tasks:
- name: get osd numbers
shell: "if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi"
register: osd_ids
changed_when: false
when: not containerized_deployment
- name: start ceph osds with upstart
service:
name: ceph-osd-all
state: started
when: ansible_service_mgr == 'upstart'
- name: start ceph osds with sysvinit
service:
name: ceph
state: started
when: ansible_service_mgr == 'sysvinit'
- name: start ceph osds with systemd
service:
name: ceph-osd@{{item}}
state: started
enabled: yes
with_items: "{{ osd_ids.stdout_lines }}"
when:
- ansible_service_mgr == 'systemd'
- not containerized_deployment
- name: restart containerized ceph osds with systemd
service:
name: ceph-osd@{{ item | basename }}
state: restarted
enabled: yes
with_items: "{{ devices }}"
when:
- ansible_service_mgr == 'systemd'
- containerized_deployment
- name: waiting for clean pgs...
command: ceph --cluster "{{ cluster }}" -s --format json
register: ceph_health_post
until: >
((ceph_health_post.stdout | from_json).pgmap.pgs_by_state | length) == 1
and
(ceph_health_post.stdout | from_json).pgmap.pgs_by_state.0.state_name == "active+clean"
delegate_to: "{{ groups[mon_group_name][0] }}"
retries: "{{ health_osd_check_retries }}"
delay: "{{ health_osd_check_delay }}"
when:
- not containerized_deployment
- name: container - waiting for clean pgs...
command: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s --format json"
register: ceph_health_post
until: >
((ceph_health_post.stdout | from_json).pgmap.pgs_by_state | length) == 1
and
(ceph_health_post.stdout | from_json).pgmap.pgs_by_state.0.state_name == "active+clean"
delegate_to: "{{ groups[mon_group_name][0] }}"
retries: "{{ health_osd_check_retries }}"
delay: "{{ health_osd_check_delay }}"
when:
- containerized_deployment
- name: unset osd flags
hosts:
- "{{ mon_group_name|default('mons') }}"
become: True
roles:
- ceph-defaults
tasks:
- name: unset osd flags
command: ceph osd unset {{ item }} --cluster {{ cluster }}
with_items:
- noout
- noscrub
- nodeep-scrub
delegate_to: "{{ groups[mon_group_name][0] }}"
when: not containerized_deployment
- name: unset containerized osd flags
command: |
docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph osd unset {{ item }} --cluster {{ cluster }}
with_items:
- noout
- noscrub
- nodeep-scrub
delegate_to: "{{ groups[mon_group_name][0] }}"
when: containerized_deployment
- name: get osd versions
command: ceph --cluster {{ cluster }} versions
register: ceph_versions
delegate_to: "{{ groups[mon_group_name][0] }}"
when: not containerized_deployment
- name: containers - get osd versions
command: |
docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} versions
register: ceph_versions
delegate_to: "{{ groups[mon_group_name][0] }}"
when: containerized_deployment
- name: set_fact ceph_versions_osd
set_fact:
ceph_versions_osd: "{{ (ceph_versions.stdout|from_json).osd }}"
delegate_to: "{{ groups[mon_group_name][0] }}"
# length == 1 means there is a single osds versions entry
# thus all the osds are running the same version
- name: complete osds upgrade
command: ceph --cluster {{ cluster }} osd require-osd-release luminous
delegate_to: "{{ groups[mon_group_name][0] }}"
when:
- not containerized_deployment
- (ceph_versions.stdout|from_json).osd | length == 1
- ceph_versions_osd | string | search("ceph version 12")
- name: containers - complete osds upgrade
command: |
docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} osd require-osd-release luminous
delegate_to: "{{ groups[mon_group_name][0] }}"
when:
- containerized_deployment
- (ceph_versions.stdout|from_json).osd | length == 1
- ceph_versions_osd | string | search("ceph version 12")
- name: upgrade ceph mdss cluster
vars:
upgrade_ceph_packages: True
hosts:
- "{{ mds_group_name|default('mdss') }}"
serial: 1
become: True
pre_tasks:
- name: stop ceph mdss with upstart
service:
name: ceph-mds
state: stopped
args: id={{ ansible_hostname }}
when: ansible_service_mgr == 'upstart'
- name: stop ceph mdss with sysvinit
service:
name: ceph
state: stopped
args: mds
when: ansible_service_mgr == 'sysvinit'
- name: stop ceph mdss with systemd
service:
name: ceph-mds@{{ ansible_hostname }}
state: stopped
enabled: yes
when:
- ansible_service_mgr == 'systemd'
- not containerized_deployment
roles:
- ceph-defaults
- { role: ceph-common, when: not containerized_deployment }
- { role: ceph-docker-common, when: containerized_deployment }
- ceph-mds
post_tasks:
- name: start ceph mdss with upstart
service:
name: ceph-mds
state: started
args: id={{ ansible_hostname }}
when: ansible_service_mgr == 'upstart'
- name: start ceph mdss with sysvinit
service:
name: ceph
state: started
args: mds
when: ansible_service_mgr == 'sysvinit'
- name: start ceph mdss with systemd
service:
name: ceph-mds@{{ ansible_hostname }}
state: started
enabled: yes
when:
- ansible_service_mgr == 'systemd'
- not containerized_deployment
- name: restart ceph mdss
service:
name: ceph-mds@{{ ansible_hostname }}
state: restarted
enabled: yes
when:
- ansible_service_mgr == 'systemd'
- containerized_deployment
- name: upgrade ceph rgws cluster
vars:
upgrade_ceph_packages: True
hosts:
- "{{ rgw_group_name|default('rgws') }}"
serial: 1
become: True
pre_tasks:
- name: stop ceph rgws with upstart
service:
name: ceph-radosgw
state: stopped
when: ansible_service_mgr == 'upstart'
- name: stop ceph rgws with sysvinit
service:
name: radosgw
state: stopped
when: ansible_service_mgr == 'sysvinit'
- name: stop ceph rgws with systemd
service:
name: ceph-radosgw@rgw.{{ ansible_hostname }}
state: stopped
enabled: yes
when:
- ansible_service_mgr == 'systemd'
- not containerized_deployment
roles:
- ceph-defaults
- { role: ceph-common, when: not containerized_deployment }
- { role: ceph-docker-common, when: containerized_deployment }
- ceph-rgw
post_tasks:
- name: start ceph rgws with upstart
service:
name: ceph-radosgw
state: started
when: ansible_service_mgr == 'upstart'
- name: start ceph rgws with sysvinit
service:
name: radosgw
state: started
when: ansible_service_mgr == 'sysvinit'
- name: start ceph rgws with systemd
service:
name: ceph-radosgw@rgw.{{ ansible_hostname }}
state: started
enabled: yes
when:
- ansible_service_mgr == 'systemd'
- not containerized_deployment
- name: restart containerized ceph rgws with systemd
service:
name: ceph-radosgw@rgw.{{ ansible_hostname }}
state: restarted
enabled: yes
when:
- ansible_service_mgr == 'systemd'
- containerized_deployment
- name: upgrade ceph client node
vars:
upgrade_ceph_packages: True
hosts:
- "{{ client_group_name|default('clients') }}"
serial: 1
become: True
roles:
- ceph-defaults
- ceph-common
- ceph-client