ceph-mgr: create keys for MGRs

Add code in ceph-mgr for creating a keyring for manager in so that
managers can be deployed on a separate node too.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1552210

Signed-off-by: Rishabh Dave <ridave@redhat.com>
(cherry picked from commit 56bfec7c58)

group_vars/mgrs.yml.sample

@@ -15,6 +15,7 @@ dummy:
 # distributed on MGR nodes. Setting 'copy_admin_key' to 'true'
 # will copy the admin key to the /etc/ceph/ directory
 #copy_admin_key: false
+#mgr_secret: 'mgr_secret'
 
 
 ###########

group_vars/mons.yml.sample

@@ -17,7 +17,6 @@ dummy:
 # ACTIVATE BOTH FSID AND MONITOR_SECRET VARIABLES FOR NON-VAGRANT DEPLOYMENT
 #monitor_secret: "{{ monitor_keyring.stdout }}"
 #admin_secret: 'admin_secret'
-#mgr_secret: 'mgr_secret'
 
 # Secure your cluster
 # This will set the following flags on all the pools:

roles/ceph-mgr/defaults/main.yml

@@ -7,6 +7,7 @@
 # distributed on MGR nodes. Setting 'copy_admin_key' to 'true'
 # will copy the admin key to the /etc/ceph/ directory
 copy_admin_key: false
+mgr_secret: 'mgr_secret'
 
 
 ###########

roles/ceph-mgr/tasks/common.yml

@@ -7,19 +7,76 @@
     group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
     mode: "0755"
 
-- name: copy ceph keyring(s) if needed
+- name: fetch ceph mgr keyring
+  ceph_key:
+    name: "mgr.{{ ansible_hostname }}"
+    state: present
+    caps:
+      mon: allow profile mgr
+      osd: allow *
+      mds: allow *
+    cluster: "{{ cluster }}"
+    secret: "{{ (mgr_secret != 'mgr_secret') | ternary(mgr_secret, omit) }}"
+    owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+    group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+    mode: "0400"
+    dest: "/var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring"
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+  when: groups.get(mgr_group_name, []) | length == 0 # the key is present already since one of the mons created it in "create ceph mgr keyring(s)"
+
+- name: create and copy keyrings
+  when: groups.get(mgr_group_name, []) | length > 0
+  block:
+    - name: create ceph mgr keyring(s) on a mon node
+      ceph_key:
+        name: "mgr.{{ hostvars[item]['ansible_hostname'] }}"
+        state: present
+        caps:
+          mon: allow profile mgr
+          osd: allow *
+          mds: allow *
+        cluster: "{{ cluster }}"
+        secret: "{{ (mgr_secret != 'mgr_secret') | ternary(mgr_secret, omit) }}"
+        owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+        group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+        mode: "0400"
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+      with_items: "{{ groups.get(mgr_group_name, []) }}"
+      run_once: True
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+
+    - name: copy ceph mgr key(s) from mon node to the ansible server
+      fetch:
+        src: "{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
+        dest: "{{ fetch_directory }}/{{ fsid }}/{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
+        flat: yes
+      with_items: "{{ groups.get(mgr_group_name, []) }}"
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+
+    - name: copy ceph keyring(s) to mgr node
       copy:
-    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.name }}"
+        src: "{{ fetch_directory }}/{{ fsid }}/etc/ceph/{{ cluster }}.mgr.{{ ansible_hostname }}.keyring"
-    dest: "{{ item.dest }}"
+        dest: "/var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring"
+        owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+        group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+        mode: "{{ ceph_keyring_permissions }}"
+      when: cephx
+
+- name: copy ceph keyring(s) if needed
+  copy:
+    src: "{{ fetch_directory }}/{{ fsid }}/etc/ceph/{{ cluster }}.client.admin.keyring"
+    dest: "/etc/ceph/{{ cluster }}.client.admin.keyring"
     owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
     group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
     mode: "{{ ceph_keyring_permissions }}"
-  with_items:
-    - { name: "/etc/ceph/{{ cluster }}.mgr.{{ ansible_hostname }}.keyring", dest: "/var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring", copy_key: true }
-    - { name: "/etc/ceph/{{ cluster }}.client.admin.keyring", dest: "/etc/ceph/{{ cluster }}.client.admin.keyring", copy_key: "{{ copy_admin_key }}" }
   when:
     - cephx
-    - item.copy_key|bool
+    - groups.get(mgr_group_name, []) | length > 0
+    - copy_admin_key | bool
 
 - name: set mgr key permissions
   file:

roles/ceph-mon/defaults/main.yml

@@ -9,7 +9,6 @@ mon_group_name: mons
 # ACTIVATE BOTH FSID AND MONITOR_SECRET VARIABLES FOR NON-VAGRANT DEPLOYMENT
 monitor_secret: "{{ monitor_keyring.stdout }}"
 admin_secret: 'admin_secret'
-mgr_secret: 'mgr_secret'
 
 # Secure your cluster
 # This will set the following flags on all the pools:

roles/ceph-mon/tasks/ceph_keys.yml

@@ -58,22 +58,6 @@
     - groups.get(restapi_group_name, []) | length > 0
     - inventory_hostname == groups[mon_group_name]|last
 
-- name: create ceph mgr keyring(s) when mon is not containerized
-  ceph_key:
-    name: "mgr.{{ hostvars[item]['ansible_hostname'] }}"
-    state: present
-    caps:
-      mon: allow profile mgr
-      osd: allow *
-      mds: allow *
-    cluster: "{{ cluster }}"
-    secret: "{{ (mgr_secret != 'mgr_secret') | ternary(mgr_secret, omit) }}"
-  when:
-    - cephx
-    - groups.get(mgr_group_name, []) | length > 0
-    - inventory_hostname == groups[mon_group_name]|last
-  with_items: "{{ groups.get(mgr_group_name, []) }}"
-
 # once this gets backported github.com/ceph/ceph/pull/20983
 # we will be able to remove these 2 tasks below
 - name: find ceph keys
@@ -109,6 +93,7 @@
     - /var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring
     - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring
     - /var/lib/ceph/bootstrap-mds/{{ cluster }}.keyring
+    - /etc/ceph/{{ cluster }}.client.admin.keyring
     - "{{ bootstrap_rbd_keyring | default([]) }}"
   when:
     - cephx