mirror of https://github.com/ceph/ceph-ansible.git
446 lines
13 KiB
YAML
446 lines
13 KiB
YAML
---
|
|
# This playbook does a rolling update for all the Ceph services
|
|
#
|
|
# The value of 'serial:' adjusts the number of servers to be updated simultaneously.
|
|
# We recommend a value of 1, which means hosts of a group (e.g: monitor) will be
|
|
# upgraded one by one. It is really crucial for the update process to happen
|
|
# in a serialized fashion. DO NOT CHANGE THIS VALUE.
|
|
#
|
|
# The four roles that apply to the ceph hosts will be applied: ceph-common,
|
|
# ceph-mon, ceph-osd and ceph-mds. So any changes to configuration, package updates, etc,
|
|
# will be applied as part of the rolling update process.
|
|
#
|
|
|
|
# /!\ DO NOT FORGET TO CHANGE THE RELEASE VERSION FIRST! /!\
|
|
|
|
- name: confirm whether user really meant to upgrade the cluster
|
|
hosts: localhost
|
|
|
|
vars_prompt:
|
|
- name: ireallymeanit
|
|
prompt: Are you sure you want to upgrade the cluster?
|
|
default: 'no'
|
|
private: no
|
|
|
|
tasks:
|
|
- name: exit playbook, if user did not mean to upgrade cluster
|
|
fail:
|
|
msg: >
|
|
"Exiting rolling_update.yml playbook, cluster was NOT upgraded.
|
|
To upgrade the cluster, either say 'yes' on the prompt or
|
|
use `-e ireallymeanit=yes` on the command line when
|
|
invoking the playbook"
|
|
when: ireallymeanit != 'yes'
|
|
|
|
|
|
- name: gather facts and check the init system
|
|
|
|
hosts:
|
|
- "{{ mon_group_name|default('mons') }}"
|
|
- "{{ osd_group_name|default('osds') }}"
|
|
- "{{ mds_group_name|default('mdss') }}"
|
|
- "{{ rgw_group_name|default('rgws') }}"
|
|
- "{{ mgr_group_name|default('mgrs') }}"
|
|
|
|
become: True
|
|
tasks:
|
|
- debug: msg="gather facts on all Ceph hosts for following reference"
|
|
|
|
- set_fact: rolling_update=true
|
|
|
|
- name: upgrade ceph mon cluster
|
|
|
|
vars:
|
|
health_mon_check_retries: 5
|
|
health_mon_check_delay: 10
|
|
upgrade_ceph_packages: True
|
|
|
|
hosts:
|
|
- "{{ mon_group_name|default('mons') }}"
|
|
|
|
serial: 1
|
|
become: True
|
|
|
|
pre_tasks:
|
|
- name: set mon_host_count
|
|
set_fact: mon_host_count={{ groups[mon_group_name] | length }}
|
|
|
|
- debug: msg="WARNING - upgrading a ceph cluster with only one monitor node ({{ inventory_hostname }})"
|
|
when: mon_host_count | int == 1
|
|
|
|
- name: stop ceph mons with upstart
|
|
service:
|
|
name: ceph-mon
|
|
state: stopped
|
|
args: id={{ ansible_hostname }}
|
|
when: ansible_service_mgr == 'upstart'
|
|
|
|
- name: stop ceph mons with sysvinit
|
|
service:
|
|
name: ceph
|
|
state: stopped
|
|
when: ansible_service_mgr == 'sysvinit'
|
|
|
|
- name: stop ceph mons with systemd
|
|
service:
|
|
name: ceph-mon@{{ ansible_hostname }}
|
|
state: stopped
|
|
enabled: yes
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- not containerized_deployment
|
|
|
|
roles:
|
|
- ceph-mon
|
|
|
|
post_tasks:
|
|
- name: start ceph mons with upstart
|
|
service:
|
|
name: ceph-mon
|
|
state: started
|
|
args: id={{ ansible_hostname }}
|
|
when: ansible_service_mgr == 'upstart'
|
|
|
|
- name: start ceph mons with sysvinit
|
|
service:
|
|
name: ceph
|
|
state: started
|
|
when: ansible_service_mgr == 'sysvinit'
|
|
|
|
- name: start ceph mons with systemd
|
|
service:
|
|
name: ceph-mon@{{ ansible_hostname }}
|
|
state: started
|
|
enabled: yes
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- not containerized_deployment
|
|
|
|
- name: restart containerized ceph mons with systemd
|
|
service:
|
|
name: ceph-mon@{{ ansible_hostname }}
|
|
state: restarted
|
|
enabled: yes
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- containerized_deployment
|
|
|
|
- name: set mon_host_count
|
|
set_fact: mon_host_count={{ groups[mon_group_name] | length }}
|
|
|
|
- name: select a running monitor if multiple monitors
|
|
set_fact: mon_host={{ item }}
|
|
with_items: "{{ groups[mon_group_name] }}"
|
|
when:
|
|
- mon_host_count | int > 1
|
|
- item != inventory_hostname
|
|
|
|
- name: select first monitor if only one monitor
|
|
set_fact: mon_host={{ item }}
|
|
with_items: "{{ groups[mon_group_name][0] }}"
|
|
when:
|
|
- mon_host_count | int == 1
|
|
|
|
- name: waiting for the monitor to join the quorum...
|
|
shell: |
|
|
ceph -s --cluster {{ cluster }} | grep election | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }}
|
|
register: result
|
|
until: result.rc == 0
|
|
retries: "{{ health_mon_check_retries }}"
|
|
delay: "{{ health_mon_check_delay }}"
|
|
delegate_to: "{{ mon_host }}"
|
|
when: not containerized_deployment
|
|
|
|
- name: waiting for the containerized monitor to join the quorum...
|
|
shell: |
|
|
docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph -s --cluster {{ cluster }} | grep quorum | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }}
|
|
register: result
|
|
until: result.rc == 0
|
|
retries: "{{ health_mon_check_retries }}"
|
|
delay: "{{ health_mon_check_delay }}"
|
|
delegate_to: "{{ mon_host }}"
|
|
when: containerized_deployment
|
|
|
|
|
|
- name: upgrade ceph osds cluster
|
|
|
|
vars:
|
|
health_osd_check_retries: 40
|
|
health_osd_check_delay: 30
|
|
upgrade_ceph_packages: True
|
|
|
|
hosts:
|
|
- "{{ osd_group_name|default('osds') }}"
|
|
|
|
serial: 1
|
|
become: True
|
|
|
|
pre_tasks:
|
|
- name: set osd flags
|
|
command: ceph osd set {{ item }} --cluster {{ cluster }}
|
|
with_items:
|
|
- noout
|
|
- noscrub
|
|
- nodeep-scrub
|
|
delegate_to: "{{ groups[mon_group_name][0] }}"
|
|
when: not containerized_deployment
|
|
|
|
- name: set containerized osd flags
|
|
command: |
|
|
docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph osd set {{ item }} --cluster {{ cluster }}
|
|
with_items:
|
|
- noout
|
|
- noscrub
|
|
- nodeep-scrub
|
|
delegate_to: "{{ groups[mon_group_name][0] }}"
|
|
when: containerized_deployment
|
|
|
|
- name: get osd numbers
|
|
shell: "if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi"
|
|
register: osd_ids
|
|
changed_when: false
|
|
when: not containerized_deployment
|
|
|
|
- name: stop ceph osds with upstart
|
|
service:
|
|
name: ceph-osd-all
|
|
state: stopped
|
|
when: ansible_service_mgr == 'upstart'
|
|
|
|
- name: stop ceph osds with sysvinit
|
|
service:
|
|
name: ceph
|
|
state: stopped
|
|
when: ansible_service_mgr == 'sysvinit'
|
|
|
|
- name: stop ceph osds with systemd
|
|
service:
|
|
name: ceph-osd@{{item}}
|
|
state: stopped
|
|
enabled: yes
|
|
with_items: "{{ osd_ids.stdout_lines }}"
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- not containerized_deployment
|
|
|
|
roles:
|
|
- ceph-osd
|
|
|
|
post_tasks:
|
|
- name: get osd numbers
|
|
shell: "if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi"
|
|
register: osd_ids
|
|
changed_when: false
|
|
when: not containerized_deployment
|
|
|
|
- name: start ceph osds with upstart
|
|
service:
|
|
name: ceph-osd-all
|
|
state: started
|
|
when: ansible_service_mgr == 'upstart'
|
|
|
|
- name: start ceph osds with sysvinit
|
|
service:
|
|
name: ceph
|
|
state: started
|
|
when: ansible_service_mgr == 'sysvinit'
|
|
|
|
- name: start ceph osds with systemd
|
|
service:
|
|
name: ceph-osd@{{item}}
|
|
state: started
|
|
enabled: yes
|
|
with_items: "{{ osd_ids.stdout_lines }}"
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- not containerized_deployment
|
|
|
|
- name: restart containerized ceph osds with systemd
|
|
service:
|
|
name: ceph-osd@{{ item | basename }}
|
|
state: restarted
|
|
enabled: yes
|
|
with_items: "{{ ceph_osd_docker_devices }}"
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- containerized_deployment
|
|
|
|
- name: waiting for clean pgs...
|
|
shell: |
|
|
test "$(ceph pg stat --cluster {{ cluster }} | tr , '\n' | sed 's/^.*pgs: //' | awk '/active.clean/ { SUM += $1 } END {print SUM}')" -eq "$(ceph pg stat --cluster {{ cluster }} | sed 's/pgs.*//;s/^.*://;s/ //')" && ceph health --cluster {{ cluster }} | egrep -sq "HEALTH_OK|HEALTH_WARN"
|
|
register: result
|
|
until: result.rc == 0
|
|
retries: "{{ health_osd_check_retries }}"
|
|
delay: "{{ health_osd_check_delay }}"
|
|
delegate_to: "{{ groups[mon_group_name][0] }}"
|
|
when: not containerized_deployment
|
|
|
|
- name: container - waiting for clean pgs...
|
|
shell: |
|
|
test "$(docker exec ceph-osd-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph pg stat --cluster {{ cluster }} | sed 's/^.*pgs://;s/active+clean.*//;s/ //')" -eq "$(docker exec {{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph pg stat --cluster {{ cluster }} | sed 's/pgs.*//;s/^.*://;s/ //')" && docker exec {{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph health --cluster {{ cluster }} | egrep -sq "HEALTH_OK|HEALTH_WARN"
|
|
register: result
|
|
until: result.rc == 0
|
|
retries: "{{ health_osd_check_retries }}"
|
|
delay: "{{ health_osd_check_delay }}"
|
|
delegate_to: "{{ groups[mon_group_name][0] }}"
|
|
when: containerized_deployment
|
|
|
|
- name: unset osd flags
|
|
command: ceph osd unset {{ item }} --cluster {{ cluster }}
|
|
with_items:
|
|
- noout
|
|
- noscrub
|
|
- nodeep-scrub
|
|
delegate_to: "{{ groups[mon_group_name][0] }}"
|
|
when: not containerized_deployment
|
|
|
|
- name: unset containerized osd flags
|
|
command: |
|
|
docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph osd unset {{ item }} --cluster {{ cluster }}
|
|
with_items:
|
|
- noout
|
|
- noscrub
|
|
- nodeep-scrub
|
|
delegate_to: "{{ groups[mon_group_name][0] }}"
|
|
when: containerized_deployment
|
|
|
|
|
|
- name: upgrade ceph mdss cluster
|
|
|
|
vars:
|
|
upgrade_ceph_packages: True
|
|
|
|
hosts:
|
|
- "{{ mds_group_name|default('mdss') }}"
|
|
|
|
serial: 1
|
|
become: True
|
|
|
|
pre_tasks:
|
|
- name: stop ceph mdss with upstart
|
|
service:
|
|
name: ceph-mds
|
|
state: stopped
|
|
args: id={{ ansible_hostname }}
|
|
when: ansible_service_mgr == 'upstart'
|
|
|
|
- name: stop ceph mdss with sysvinit
|
|
service:
|
|
name: ceph
|
|
state: stopped
|
|
args: mds
|
|
when: ansible_service_mgr == 'sysvinit'
|
|
|
|
- name: stop ceph mdss with systemd
|
|
service:
|
|
name: ceph-mds@{{ ansible_hostname }}
|
|
state: stopped
|
|
enabled: yes
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- not containerized_deployment
|
|
|
|
roles:
|
|
- ceph-mds
|
|
|
|
post_tasks:
|
|
- name: start ceph mdss with upstart
|
|
service:
|
|
name: ceph-mds
|
|
state: started
|
|
args: id={{ ansible_hostname }}
|
|
when: ansible_service_mgr == 'upstart'
|
|
|
|
- name: start ceph mdss with sysvinit
|
|
service:
|
|
name: ceph
|
|
state: started
|
|
args: mds
|
|
when: ansible_service_mgr == 'sysvinit'
|
|
|
|
- name: start ceph mdss with systemd
|
|
service:
|
|
name: ceph-mds@{{ ansible_hostname }}
|
|
state: started
|
|
enabled: yes
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- not containerized_deployment
|
|
|
|
- name: restart ceph mdss
|
|
service:
|
|
name: ceph-mds@{{ ansible_hostname }}
|
|
state: restarted
|
|
enabled: yes
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- containerized_deployment
|
|
|
|
|
|
- name: upgrade ceph rgws cluster
|
|
|
|
vars:
|
|
upgrade_ceph_packages: True
|
|
|
|
hosts:
|
|
- "{{ rgw_group_name|default('rgws') }}"
|
|
|
|
serial: 1
|
|
become: True
|
|
|
|
pre_tasks:
|
|
- name: stop ceph rgws with upstart
|
|
service:
|
|
name: ceph-radosgw
|
|
state: stopped
|
|
when: ansible_service_mgr == 'upstart'
|
|
|
|
- name: stop ceph rgws with sysvinit
|
|
service:
|
|
name: radosgw
|
|
state: stopped
|
|
when: ansible_service_mgr == 'sysvinit'
|
|
|
|
- name: stop ceph rgws with systemd
|
|
service:
|
|
name: ceph-radosgw@rgw.{{ ansible_hostname }}
|
|
state: stopped
|
|
enabled: yes
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- not containerized_deployment
|
|
|
|
roles:
|
|
- ceph-rgw
|
|
|
|
post_tasks:
|
|
- name: start ceph rgws with upstart
|
|
service:
|
|
name: ceph-radosgw
|
|
state: started
|
|
when: ansible_service_mgr == 'upstart'
|
|
|
|
- name: start ceph rgws with sysvinit
|
|
service:
|
|
name: radosgw
|
|
state: started
|
|
when: ansible_service_mgr == 'sysvinit'
|
|
|
|
- name: start ceph rgws with systemd
|
|
service:
|
|
name: ceph-radosgw@rgw.{{ ansible_hostname }}
|
|
state: started
|
|
enabled: yes
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- not containerized_deployment
|
|
|
|
- name: restart containerized ceph rgws with systemd
|
|
service:
|
|
name: ceph-rgw@{{ ansible_hostname }}
|
|
state: restarted
|
|
enabled: yes
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
- containerized_deployment
|