From 68f1f99ee9e1c6c9f1728240ceaa15db1202442f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 28 Sep 2017 14:46:26 +0200 Subject: [PATCH 1/2] update: nicer way to wait for clean pgs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit More comprhensive and friendly to read. Signed-off-by: Sébastien Han --- infrastructure-playbooks/rolling_update.yml | 45 +++++++-------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml index ba8d2bac7..6907c35db 100644 --- a/infrastructure-playbooks/rolling_update.yml +++ b/infrastructure-playbooks/rolling_update.yml @@ -273,46 +273,31 @@ - ansible_service_mgr == 'systemd' - containerized_deployment - - name: get pg num - shell: | - ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])' - delegate_to: "{{ groups[mon_group_name][0] }}" - register: num_pgs - when: - - not containerized_deployment - - name: waiting for clean pgs... - shell: | - test "[""$(ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])')""]" = "$(ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print [ i["count"] for i in json.load(sys.stdin)["pgmap"]["pgs_by_state"] if i["state_name"] == "active+clean"]')" - register: result - until: result.rc == 0 + command: ceph --cluster "{{ cluster }}" -s --format json + register: ceph_health_post + until: > + ((ceph_health_post.stdout | from_json).pgmap.pgs_by_state | length) == 1 + and + (ceph_health_post.stdout | from_json).pgmap.pgs_by_state.0.state_name == "active+clean" + delegate_to: "{{ groups[mon_group_name][0] }}" retries: "{{ health_osd_check_retries }}" delay: "{{ health_osd_check_delay }}" - delegate_to: "{{ groups[mon_group_name][0] }}" when: - not containerized_deployment - - num_pgs.stdout != "0" - - - name: container - get pg num - shell: | - docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])' - delegate_to: "{{ groups[mon_group_name][0] }}" - register: container_num_pgs - when: - - containerized_deployment - name: container - waiting for clean pgs... - shell: | - test "[""$(docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])')""]" = "$(docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print [ i["count"] for i in json.load(sys.stdin)["pgmap"]["pgs_by_state"] if i["state_name"] == "active+clean"]')" - register: result - until: result.rc == 0 + command: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster "{{ cluster }}" -s --format json" + register: ceph_health_post + until: > + ((ceph_health_post.stdout | from_json).pgmap.pgs_by_state | length) == 1 + and + (ceph_health_post.stdout | from_json).pgmap.pgs_by_state.0.state_name == "active+clean" + delegate_to: "{{ groups[mon_group_name][0] }}" retries: "{{ health_osd_check_retries }}" delay: "{{ health_osd_check_delay }}" - delegate_to: "{{ groups[mon_group_name][0] }}" when: - - containerized_deployment - - container_num_pgs.stdout != "0" - + - not containerized_deployment - name: unset osd flags From bd5471b940b124a1441065cdb2e033db4841c95d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 28 Sep 2017 15:27:27 +0200 Subject: [PATCH 2/2] update: complete luminous upgrade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once we complete the upgrade to Luminous, we must issue a specific command. For more info read: http://ceph.com/community/new-luminous-upgrade-complete/ Signed-off-by: Sébastien Han --- infrastructure-playbooks/rolling_update.yml | 23 +++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml index 6907c35db..93f1d5d83 100644 --- a/infrastructure-playbooks/rolling_update.yml +++ b/infrastructure-playbooks/rolling_update.yml @@ -287,7 +287,7 @@ - not containerized_deployment - name: container - waiting for clean pgs... - command: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster "{{ cluster }}" -s --format json" + command: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s --format json" register: ceph_health_post until: > ((ceph_health_post.stdout | from_json).pgmap.pgs_by_state | length) == 1 @@ -297,7 +297,7 @@ retries: "{{ health_osd_check_retries }}" delay: "{{ health_osd_check_delay }}" when: - - not containerized_deployment + - containerized_deployment - name: unset osd flags @@ -329,6 +329,25 @@ delegate_to: "{{ groups[mon_group_name][0] }}" when: containerized_deployment + - name: get osd versions + command: ceph --cluster {{ cluster }} versions + register: ceph_versions + delegate_to: "{{ groups[mon_group_name][0] }}" + + - name: set_fact ceph_versions_osd + set_fact: + ceph_versions_osd: "{{ (ceph_versions.stdout|from_json).osd }}" + delegate_to: "{{ groups[mon_group_name][0] }}" + + # length == 1 means there is a single osds versions entry + # thus all the osds are running the same version + - name: complete osds upgrade + command: ceph --cluster {{ cluster }} osd require-osd-release luminous + delegate_to: "{{ groups[mon_group_name][0] }}" + when: + - (ceph_versions.stdout|from_json).osd | length == 1 + - ceph_versions_osd | string | search("ceph version 12") + - name: upgrade ceph mdss cluster