From d73dde0fc7e34c61e2051a2f355177a6243a01af Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Tue, 12 Oct 2021 16:01:20 +0200 Subject: [PATCH] adopt: fix rbd mirror adoption The rbd mirroring is broken because cephadm doesn't bindmount /etc/ceph anymore. It means the keyrings and ceph config file aren't available after the migration. The idea here is to remove the current rbd mirror peer and add it back to the mon config store so we aren't bound to the /etc/ceph directory. Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1967440 Signed-off-by: Guillaume Abrioux (cherry picked from commit 9c794aa9bcfab177f437f503bda0ff37dceca319) --- infrastructure-playbooks/cephadm-adopt.yml | 100 +++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/infrastructure-playbooks/cephadm-adopt.yml b/infrastructure-playbooks/cephadm-adopt.yml index cef87c5b6..1a6470c98 100644 --- a/infrastructure-playbooks/cephadm-adopt.yml +++ b/infrastructure-playbooks/cephadm-adopt.yml @@ -329,6 +329,106 @@ delegate_to: '{{ groups[mon_group_name][0] }}' when: ceph_docker_registry_auth | bool + +- name: store existing rbd mirror peers in monitor config store + hosts: "{{ rbdmirror_group_name|default('rbdmirrors') }}" + become: true + any_errors_fatal: true + gather_facts: true + tasks: + - name: store existing rbd mirror peers in monitor config store + when: ceph_rbd_mirror_configure | default(False) | bool + block: + - name: import ceph-defaults + import_role: + name: ceph-defaults + + - name: import ceph-validate + import_role: + name: ceph-validate + tasks_from: check_rbdmirror.yml + + - name: import container_binary + import_role: + name: ceph-facts + tasks_from: container_binary.yml + + - name: set_fact rbd_cmd + set_fact: + rbd_cmd: "{{ container_binary + ' run --rm --net=host -v /etc/ceph:/etc/ceph:z -v /var/lib/ceph:/var/lib/ceph:z -v /var/run/ceph:/var/run/ceph:z --entrypoint=rbd ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else 'ceph' }} --cluster {{ cluster }} -n client.rbd-mirror.{{ ansible_facts['hostname'] }} -k /etc/ceph/{{ cluster }}.client.rbd-mirror.{{ ansible_facts['hostname'] }}.keyring" + + - name: set_fact admin_rbd_cmd + set_fact: + admin_rbd_cmd: "{{ container_binary + ' run --rm --net=host -v /etc/ceph:/etc/ceph:z -v /var/lib/ceph:/var/lib/ceph:z -v /var/run/ceph:/var/run/ceph:z --entrypoint=rbd ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else 'ceph' }} --cluster {{ cluster }}" + - name: get mirror pool info + command: "{{ rbd_cmd }} mirror pool info {{ ceph_rbd_mirror_pool }} --format json" + register: mirror_pool_info + changed_when: false + + - name: set_fact mirror_peer_found + set_fact: + mirror_peer_uuid: "{{ ((mirror_pool_info.stdout | from_json | default('{}'))['peers'] | selectattr('site_name', 'match', '^'+ceph_rbd_mirror_remote_cluster+'$') | map(attribute='uuid') | list) }}" + + - name: remove current rbd mirror peer, add new peer into mon config store + when: mirror_peer_uuid | length > 0 + block: + - name: get remote user keyring + slurp: + src: "/etc/ceph/{{ ceph_rbd_mirror_remote_cluster }}.{{ ceph_rbd_mirror_remote_user }}.keyring" + register: remote_user_keyring + + - name: get quorum_status + command: "{{ cephadm_cmd }} shell --fsid {{ fsid }} -- ceph --cluster {{ cluster }} quorum_status --format json" + changed_when: false + delegate_to: "{{ groups[mon_group_name][0] }}" + register: quorum_status + run_once: true + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: set_fact mon_ip_list + set_fact: + mon_ip_list: "{{ mon_ip_list | default([]) | union([item['addr'].split(':')[0]]) }}" + loop: "{{ (quorum_status.stdout | from_json | default('{}'))['monmap']['mons'] }}" + run_once: true + + - name: remove current mirror peer + command: "{{ admin_rbd_cmd }} mirror pool peer remove {{ ceph_rbd_mirror_pool }} {{ ((mirror_pool_info.stdout | from_json | default('{}'))['peers'] | selectattr('site_name', 'match', '^'+ceph_rbd_mirror_remote_cluster+'$') | map(attribute='uuid') | list)[0] }}" + delegate_to: "{{ groups.get(mon_group_name | default('mons'))[0] }}" + changed_when: false + + - name: get remote user keyring secret + set_fact: + remote_user_keyring_secret: "{{ item.split('=', 1)[1] | trim }}" + with_items: "{{ (remote_user_keyring.content | b64decode).split('\n') }}" + when: "'key = ' in item" + + - name: create a temporary file + tempfile: + path: /etc/ceph + state: file + suffix: _ceph-ansible + register: tmp_file + delegate_to: "{{ groups.get(mon_group_name | default('mons'))[0] }}" + + - name: write secret to temporary file + copy: + dest: "{{ tmp_file.path }}" + content: "{{ remote_user_keyring_secret }}" + delegate_to: "{{ groups.get(mon_group_name | default('mons'))[0] }}" + + - name: re-add mirror peer + command: "{{ admin_rbd_cmd }} mirror pool peer add {{ ceph_rbd_mirror_pool }} {{ ceph_rbd_mirror_remote_user }}@{{ ceph_rbd_mirror_remote_cluster }} --remote-mon-host {{ ','.join(mon_ip_list) }} --remote-key-file {{ tmp_file.path }}" + delegate_to: "{{ groups.get(mon_group_name | default('mons'))[0] }}" + changed_when: false + + - name: rm temporary file + file: + path: "{{ tmp_file.path }}" + state: absent + delegate_to: "{{ groups.get(mon_group_name | default('mons'))[0] }}" + + - name: adopt ceph mon daemons hosts: "{{ mon_group_name|default('mons') }}" serial: 1