nfs: fix nfs with external ceph cluster support

This commit refact and fix the nfs deployment with external ceph cluster
support.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1814942

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>

group_vars/clients.yml.sample

@@ -75,5 +75,3 @@ dummy:
 #  - { name: client.test, caps: { mon: "profile rbd", osd: "allow class-read object_prefix rbd_children, profile rbd pool=test" },  mode: "{{ ceph_keyring_permissions }}" }
 #  - { name: client.test2, caps: { mon: "profile rbd", osd: "allow class-read object_prefix rbd_children, profile rbd pool=test2" },  mode: "{{ ceph_keyring_permissions }}" }
 
-#ceph_nfs_ceph_user: { name: client.rgw.nfs0, key: 'SECRET==', caps: { mon: "allow rw", osd: "allow rwx" }, mode: "{{ ceph_keyring_permissions }}" }
-

roles/ceph-client/defaults/main.yml

@@ -66,5 +66,3 @@ pools:
 keys:
   - { name: client.test, caps: { mon: "profile rbd", osd: "allow class-read object_prefix rbd_children, profile rbd pool=test" },  mode: "{{ ceph_keyring_permissions }}" }
   - { name: client.test2, caps: { mon: "profile rbd", osd: "allow class-read object_prefix rbd_children, profile rbd pool=test2" },  mode: "{{ ceph_keyring_permissions }}" }
-
-#ceph_nfs_ceph_user: { name: client.rgw.nfs0, key: 'SECRET==', caps: { mon: "allow rw", osd: "allow rwx" }, mode: "{{ ceph_keyring_permissions }}" }

roles/ceph-client/tasks/create_users_keys.yml

@@ -47,7 +47,6 @@
     CEPH_CONTAINER_BINARY: "{{ container_binary }}"
   with_items:
     - "{{ keys }}"
-    - "{{ ceph_nfs_ceph_user | default([]) }}"
   delegate_to: "{{ delegated_node }}"
   when:
     - cephx | bool

roles/ceph-nfs/tasks/main.yml

@@ -26,17 +26,31 @@
     - not containerized_deployment | bool
     - ansible_os_family == 'RedHat'
 
-- name: copy rgw keyring when deploying internal ganesha with external ceph cluster
-  copy:
-    src: "/etc/ceph/{{ cluster }}.{{ ceph_nfs_ceph_user }}.keyring"
-    dest: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring"
-    mode: '0600'
-    owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
-    group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
-    remote_src: true
+- name: nfs with external ceph cluster task related
   when:
     - groups.get(mon_group_name, []) | length == 0
     - ceph_nfs_ceph_user is defined
+  block:
+    - name: create keyring directory
+      file:
+        path: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}"
+        state: directory
+        owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+        group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+        mode: "0755"
+      when:
+
+    - name: get client cephx keys
+      copy:
+        dest: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring"
+        content: "{{ item.content | b64decode }}"
+        mode: "{{ item.item.get('mode', '0600') }}"
+        owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+        group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+      with_items: "{{ hostvars[groups['_filtered_clients'][0]]['slurp_client_keys']['results'] | default([]) }}"
+      when:
+        - not item.get('skipped', False)
+        - item.item.name == 'client.' + ceph_nfs_ceph_user
 
 - name: include start_nfs.yml
   import_tasks: start_nfs.yml

roles/ceph-validate/tasks/main.yml

@@ -261,12 +261,3 @@
     - keys | length > 0
     - item.key is defined
     - item.key is not match("^[a-zA-Z0-9+/]{38}==$")
-
-- name: validate ceph_nfs_ceph_user key format
-  fail:
-    msg: '{{ ceph_nfs_ceph_user.name }} key format invalid'
-  when:
-    - client_group_name in group_names
-    - ceph_nfs_ceph_user is defined
-    - ceph_nfs_ceph_user.key is defined
-    - ceph_nfs_ceph_user.key is not match("^[a-zA-Z0-9+/]{38}==$")

site-container.yml.sample

@@ -258,6 +258,44 @@
             status: "Complete"
             end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
+- hosts: clients
+  become: True
+  gather_facts: false
+  any_errors_fatal: true
+  tasks:
+    # pre-tasks for following imports -
+    - name: set ceph client install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_client:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
+    - import_role:
+        name: ceph-defaults
+      tags: ['ceph_update_config']
+    - import_role:
+        name: ceph-facts
+      tags: ['ceph_update_config']
+    - import_role:
+        name: ceph-handler
+      tags: ['ceph_update_config']
+    - import_role:
+        name: ceph-config
+      tags: ['ceph_update_config']
+    - import_role:
+        name: ceph-client
+
+    # post-tasks for preceding imports -
+    - name: set ceph client install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_client:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
 - hosts: nfss
   become: True
   gather_facts: false
@@ -334,44 +372,6 @@
             status: "Complete"
             end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
-- hosts: clients
-  become: True
-  gather_facts: false
-  any_errors_fatal: true
-  tasks:
-    # pre-tasks for following imports -
-    - name: set ceph client install 'In Progress'
-      run_once: true
-      set_stats:
-        data:
-          installer_phase_ceph_client:
-            status: "In Progress"
-            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
-
-    - import_role:
-        name: ceph-defaults
-      tags: ['ceph_update_config']
-    - import_role:
-        name: ceph-facts
-      tags: ['ceph_update_config']
-    - import_role:
-        name: ceph-handler
-      tags: ['ceph_update_config']
-    - import_role:
-        name: ceph-config
-      tags: ['ceph_update_config']
-    - import_role:
-        name: ceph-client
-
-    # post-tasks for preceding imports -
-    - name: set ceph client install 'Complete'
-      run_once: true
-      set_stats:
-        data:
-          installer_phase_ceph_client:
-            status: "Complete"
-            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
-
 - hosts:
     - iscsigws
     - iscsi-gws # for backward compatibility only!

site.yml.sample

@@ -248,6 +248,44 @@
             status: "Complete"
             end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
+- hosts: clients
+  gather_facts: false
+  become: True
+  any_errors_fatal: true
+  pre_tasks:
+    - name: set ceph client install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_client:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
+  tasks:
+    - import_role:
+        name: ceph-defaults
+      tags: ['ceph_update_config']
+    - import_role:
+        name: ceph-facts
+      tags: ['ceph_update_config']
+    - import_role:
+        name: ceph-handler
+      tags: ['ceph_update_config']
+    - import_role:
+        name: ceph-config
+      tags: ['ceph_update_config']
+    - import_role:
+        name: ceph-client
+
+  post_tasks:
+    - name: set ceph client install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_client:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
 - hosts: nfss
   gather_facts: false
   become: True
@@ -324,44 +362,6 @@
             status: "Complete"
             end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
-- hosts: clients
-  gather_facts: false
-  become: True
-  any_errors_fatal: true
-  pre_tasks:
-    - name: set ceph client install 'In Progress'
-      run_once: true
-      set_stats:
-        data:
-          installer_phase_ceph_client:
-            status: "In Progress"
-            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
-
-  tasks:
-    - import_role:
-        name: ceph-defaults
-      tags: ['ceph_update_config']
-    - import_role:
-        name: ceph-facts
-      tags: ['ceph_update_config']
-    - import_role:
-        name: ceph-handler
-      tags: ['ceph_update_config']
-    - import_role:
-        name: ceph-config
-      tags: ['ceph_update_config']
-    - import_role:
-        name: ceph-client
-
-  post_tasks:
-    - name: set ceph client install 'Complete'
-      run_once: true
-      set_stats:
-        data:
-          installer_phase_ceph_client:
-            status: "Complete"
-            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
-
 - hosts:
     - iscsigws
     - iscsi-gws # for backward compatibility only!