shrink-rbdmirror: fix presence after removal

We should add retry/delay to check the presence of the rbdmirror daemon
in the cluster status because the status takes some time to be updated.
Also the metadata.hostname isn't a good key to check because it doesn't
reflect the ansible_hostname fact. We should use metadata.id instead.

Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>

infrastructure-playbooks/shrink-rbdmirror.yml

@@ -87,7 +87,7 @@
       set_fact:
         rbdmirror_to_kill_gid: "{{ (ceph_health.stdout | from_json)['servicemap']['services']['rbd-mirror']['daemons'][item]['gid'] }}"
       with_items: "{{ rbdmirror_gids }}"
-      when: (ceph_health.stdout | from_json)['servicemap']['services']['rbd-mirror']['daemons'][item]['metadata']['hostname'] == rbdmirror_to_kill_hostname
+      when: (ceph_health.stdout | from_json)['servicemap']['services']['rbd-mirror']['daemons'][item]['metadata']['id'] == rbdmirror_to_kill_hostname
 
   tasks:
     - name: stop rbdmirror service
@@ -108,22 +108,15 @@
     - name: get servicemap details
       command: "{{ container_exec_cmd | default('') }} timeout 5 ceph --cluster {{ cluster }} -s -f json"
       register: ceph_health
-
-    - name: set_fact rbdmirror_gids
-      set_fact:
-        post_rbdmirror_gids: "{{ post_rbdmirror_gids | default([]) + [ item ] }}"
-      with_items: "{{  (ceph_health.stdout | from_json)['servicemap']['services']['rbd-mirror']['daemons'].keys() | list }}"
-      when:
+      failed_when:
         - "'rbd-mirror' in (ceph_health.stdout | from_json)['servicemap']['services'].keys() | list"
-        - item != 'summary'
+        - rbdmirror_to_kill_gid in (ceph_health.stdout | from_json)['servicemap']['services']['rbd-mirror']['daemons'].keys() | list
+      until:
+        - "'rbd-mirror' in (ceph_health.stdout | from_json)['servicemap']['services'].keys() | list"
+        - rbdmirror_to_kill_gid not in (ceph_health.stdout | from_json)['servicemap']['services']['rbd-mirror']['daemons'].keys() | list
+      when: rbdmirror_to_kill_gid is defined
+      retries: 12
+      delay: 10
 
     - name: show ceph health
       command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} -s"
-
-    - name: check presence of "{{ rbdmirror_to_kill_hostname }}"
-      fail:
-        msg: "{{ rbdmirror_to_kill_hostname }} still active!"
-      when:
-        - rbdmirror_to_kill_gid is defined
-        - post_rbdmirror_gids is defined
-        - rbdmirror_to_kill_gid in post_rbdmirror_gids