From d8ab11d2f8e08c28731f783d592507dd198b5742 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 23 Oct 2019 19:39:15 +0200 Subject: [PATCH] update: avoid skipping single mds deployment upgrade otherwise a single MDS would never be updated. Signed-off-by: Guillaume Abrioux --- infrastructure-playbooks/rolling_update.yml | 113 ++++++++++---------- 1 file changed, 58 insertions(+), 55 deletions(-) diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml index b2b033406..82388cd10 100644 --- a/infrastructure-playbooks/rolling_update.yml +++ b/infrastructure-playbooks/rolling_update.yml @@ -542,77 +542,80 @@ become: true tasks: - name: deactivate all mds rank > 0 - when: groups.get(mds_group_name, []) | length > 1 + when: groups.get(mds_group_name, []) | length > 0 block: - import_role: name: ceph-defaults - import_role: name: ceph-facts - - name: set max_mds 1 on ceph fs - command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs set {{ cephfs }} max_mds 1" - changed_when: false + - name: deactivate all mds rank > 0 if any + when: groups.get(mds_group_name, []) | length > 1 + block: + - name: set max_mds 1 on ceph fs + command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs set {{ cephfs }} max_mds 1" + changed_when: false - - name: wait until only rank 0 is up - command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs get {{ cephfs }} -f json" - changed_when: false - register: wait_rank_zero - retries: 720 - delay: 5 - until: (wait_rank_zero.stdout | from_json).mdsmap.in | length == 1 and (wait_rank_zero.stdout | from_json).mdsmap.in[0] == 0 + - name: wait until only rank 0 is up + command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs get {{ cephfs }} -f json" + changed_when: false + register: wait_rank_zero + retries: 720 + delay: 5 + until: (wait_rank_zero.stdout | from_json).mdsmap.in | length == 1 and (wait_rank_zero.stdout | from_json).mdsmap.in[0] == 0 - - name: get name of remaining active mds - command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs dump -f json" - changed_when: false - register: _mds_active_name + - name: get name of remaining active mds + command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs dump -f json" + changed_when: false + register: _mds_active_name - - name: set_fact mds_active_name - set_fact: - mds_active_name: "{{ [(_mds_active_name.stdout | from_json)['filesystems'][0]['mdsmap']['info'][item.key]['name']] }}" - with_dict: "{{ (_mds_active_name.stdout | from_json).filesystems[0]['mdsmap']['info'] }}" + - name: set_fact mds_active_name + set_fact: + mds_active_name: "{{ [(_mds_active_name.stdout | from_json)['filesystems'][0]['mdsmap']['info'][item.key]['name']] }}" + with_dict: "{{ (_mds_active_name.stdout | from_json).filesystems[0]['mdsmap']['info'] }}" + + - name: create standby_mdss group + add_host: + name: "{{ item }}" + groups: standby_mdss + ansible_host: "{{ hostvars[item]['ansible_host'] | default(omit) }}" + ansible_port: "{{ hostvars[item]['ansible_port'] | default(omit) }}" + with_items: "{{ groups[mds_group_name] | difference(mds_active_name) }}" + + - name: stop standby ceph mds + systemd: + name: "ceph-mds@{{ hostvars[item]['ansible_hostname'] }}" + state: stopped + enabled: no + delegate_to: "{{ item }}" + with_items: "{{ groups['standby_mdss'] }}" + when: groups['standby_mdss'] | default([]) | length > 0 + + # dedicated task for masking systemd unit + # somehow, having a single task doesn't work in containerized context + - name: mask systemd units for standby ceph mds + systemd: + name: "ceph-mds@{{ hostvars[item]['ansible_hostname'] }}" + masked: yes + delegate_to: "{{ item }}" + with_items: "{{ groups['standby_mdss'] }}" + when: groups['standby_mdss'] | default([]) | length > 0 + + - name: wait until all standbys mds are stopped + command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs dump -f json" + changed_when: false + register: wait_standbys_down + retries: 300 + delay: 5 + until: (wait_standbys_down.stdout | from_json).standbys | length == 0 - name: create active_mdss group add_host: - name: "{{ mds_active_name[0] }}" + name: "{{ mds_active_name[0] if mds_active_name is defined else groups.get(mds_group_name)[0] }}" groups: active_mdss ansible_host: "{{ hostvars[mds_active_name]['ansible_host'] | default(omit) }}" ansible_port: "{{ hostvars[mds_active_name]['ansible_port'] | default(omit) }}" - - name: create standby_mdss group - add_host: - name: "{{ item }}" - groups: standby_mdss - ansible_host: "{{ hostvars[item]['ansible_host'] | default(omit) }}" - ansible_port: "{{ hostvars[item]['ansible_port'] | default(omit) }}" - with_items: "{{ groups[mds_group_name] | difference(mds_active_name) }}" - - - name: stop standby ceph mds - systemd: - name: "ceph-mds@{{ hostvars[item]['ansible_hostname'] }}" - state: stopped - enabled: no - delegate_to: "{{ item }}" - with_items: "{{ groups['standby_mdss'] }}" - when: groups['standby_mdss'] | default([]) | length > 0 - - # dedicated task for masking systemd unit - # somehow, having a single task doesn't work in containerized context - - name: mask systemd units for standby ceph mds - systemd: - name: "ceph-mds@{{ hostvars[item]['ansible_hostname'] }}" - masked: yes - delegate_to: "{{ item }}" - with_items: "{{ groups['standby_mdss'] }}" - when: groups['standby_mdss'] | default([]) | length > 0 - - - name: wait until all standbys mds are stopped - command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs dump -f json" - changed_when: false - register: wait_standbys_down - retries: 300 - delay: 5 - until: (wait_standbys_down.stdout | from_json).standbys | length == 0 - - name: upgrade active mds vars: