rgw: fix multi instances scaleout in baremetal

When rgw and osd are collocated, the current workflow prevents from
scaling out the radosgw_num_instances parameter when rerunning the
playbook in baremetal deployments.

When ceph-osd notifies handlers, it means rgw handlers are triggered
too. The issue with this is that they are triggered before the role
ceph-rgw is run.
In the case a scaleout operation is expected on `radosgw_num_instances`
it causes an issue because keyrings haven't been created yet so the new
instances won't start.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1881313

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
(cherry picked from commit a802fa2810)

roles/ceph-handler/tasks/main.yml

@@ -48,3 +48,18 @@
       or inventory_hostname in groups.get(mds_group_name, [])
       or inventory_hostname in groups.get(rgw_group_name, [])
       or inventory_hostname in groups.get(rbdmirror_group_name, [])
+
+- name: rgw multi-instances related tasks
+  when:
+    - inventory_hostname in groups.get(rgw_group_name, [])
+    - handler_rgw_status | bool
+  block:
+    - name: import_role ceph-config
+      import_role:
+        name: ceph-config
+
+    - name: import_role ceph-rgw
+      import_role:
+        name: ceph-rgw
+        tasks_from: pre_requisite.yml
+      when: not containerized_deployment | bool

roles/ceph-rgw/tasks/pre_requisite.yml

@@ -1,17 +1,21 @@
 ---
-- name: create rados gateway instance keyring
-  command: ceph --cluster {{ cluster }} --name client.bootstrap-rgw --keyring /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring auth get-or-create client.rgw.{{ ansible_hostname }}.{{ item.instance_name }} osd 'allow rwx' mon 'allow rw' -o /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/keyring
-  args:
-    creates: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/keyring
-  changed_when: false
-  with_items: "{{ rgw_instances }}"
-  when: cephx | bool
-
-- name: set rados gateway instance key permissions
-  file:
-    path: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/keyring
+- name: create rgw keyrings
+  ceph_key:
+    state: present
+    name: "client.rgw.{{ ansible_hostname }}.{{ item.instance_name }}"
+    cluster: "{{ cluster }}"
+    user: "client.bootstrap-rgw"
+    user_key: /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring
+    dest: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/keyring"
+    caps:
+      osd: 'allow rwx'
+      mon: 'allow rw'
+    import_key: False
     owner: "ceph"
     group: "ceph"
     mode: "0600"
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
   with_items: "{{ rgw_instances }}"
-  when: cephx | bool
+  when: cephx | bool