nfs: support internal Ganesha with external ceph cluster

This commits allows to deploy an internal ganesha with an external ceph
cluster.

This requires to define `external_cluster_mon_ips` with a comma
separated list of external monitors.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1710358

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
(cherry picked from commit 6a6785b719)

group_vars/all.yml.sample

@@ -474,7 +474,7 @@ dummy:
 # Set this to true to enable File access via NFS.  Requires an MDS role.
 #nfs_file_gw: false
 # Set this to true to enable Object access via NFS. Requires an RGW role.
-#nfs_obj_gw: true
+#nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}"
 
 
 #############

group_vars/clients.yml.sample

@@ -57,3 +57,4 @@ dummy:
 #  - { name: client.test, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test" },  mode: "{{ ceph_keyring_permissions }}" }
 #  - { name: client.test2, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test2" },  mode: "{{ ceph_keyring_permissions }}" }
 
+#ceph_nfs_ceph_user: { name: client.rgw.nfs0, key: 'SECRET==', caps: { mon: "allow rw", osd: "allow rwx" }, mode: "{{ ceph_keyring_permissions }}" }

group_vars/rhcs.yml.sample

@@ -474,7 +474,7 @@ ceph_rhcs_version: 4
 # Set this to true to enable File access via NFS.  Requires an MDS role.
 #nfs_file_gw: false
 # Set this to true to enable Object access via NFS. Requires an RGW role.
-#nfs_obj_gw: true
+#nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}"
 
 
 #############

roles/ceph-client/defaults/main.yml

@@ -48,3 +48,5 @@ pools:
 keys:
   - { name: client.test, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test" },  mode: "{{ ceph_keyring_permissions }}" }
   - { name: client.test2, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test2" },  mode: "{{ ceph_keyring_permissions }}" }
+
+#ceph_nfs_ceph_user: { name: client.rgw.nfs0, key: 'SECRET==', caps: { mon: "allow rw", osd: "allow rwx" }, mode: "{{ ceph_keyring_permissions }}" }

roles/ceph-client/tasks/create_users_keys.yml

@@ -45,7 +45,9 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
     CEPH_CONTAINER_BINARY: "{{ container_binary }}"
-  with_items: "{{ keys }}"
+  with_items:
+    - "{{ keys }}"
+    - "{{ ceph_nfs_ceph_user | default([]) }}"
   delegate_to: "{{ delegated_node }}"
   when:
     - cephx

roles/ceph-config/templates/ceph.conf.j2

@@ -56,7 +56,7 @@ mon host = {% if nb_mon > 0 %}
 {%- endif %}
 {%- if not loop.last -%},{%- endif %}
 {%- endfor %}
-{% elif nb_mon == 0 and inventory_hostname in groups.get(client_group_name, []) %}
+{% elif nb_mon == 0 %}
 {{ external_cluster_mon_ips }}
 {% endif %}
 

roles/ceph-defaults/defaults/main.yml

@@ -466,7 +466,7 @@ handler_health_mgr_check_delay: 10
 # Set this to true to enable File access via NFS.  Requires an MDS role.
 nfs_file_gw: false
 # Set this to true to enable Object access via NFS. Requires an RGW role.
-nfs_obj_gw: true
+nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}"
 
 
 #############

roles/ceph-nfs/tasks/common.yml

@@ -12,3 +12,4 @@
   when:
     - cephx
     - item.copy_key|bool
+    - groups.get(mon_group_name, []) | length > 0

roles/ceph-nfs/tasks/main.yml

@@ -17,6 +17,7 @@
 
 - name: include create_rgw_nfs_user.yml
   import_tasks: create_rgw_nfs_user.yml
+  when: groups.get(mon_group_name, []) | length > 0
 
 # NOTE (leseb): workaround for issues with ganesha and librgw
 - name: include ganesha_selinux_fix.yml
@@ -25,5 +26,17 @@
     - not containerized_deployment
     - ansible_os_family == 'RedHat'
 
+- name: copy rgw keyring when deploying internal ganesha with external ceph cluster
+  copy:
+    src: "/etc/ceph/{{ cluster }}.{{ ceph_nfs_ceph_user.name }}.keyring"
+    dest: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring"
+    mode: '0600'
+    owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+    group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+    remote_src: true
+  when:
+    - groups.get(mon_group_name, []) | length == 0
+    - ceph_nfs_ceph_user is defined
+
 - name: include start_nfs.yml
   import_tasks: start_nfs.yml

roles/ceph-nfs/tasks/pre_requisite_container.yml

@@ -1,4 +1,6 @@
 ---
+- name: keyring related tasks
+  block:
     - name: set_fact admin_keyring
       set_fact:
         admin_keyring:
@@ -38,7 +40,10 @@
         - "{{ ceph_config_keys }}"
         - "{{ statconfig.results }}"
       when: item.1.stat.exists
+  when: groups.get(mon_group_name, []) | length > 0
 
+- name: dbus related tasks
+  block:
     - name: create dbus service file
       become: true
       copy:
@@ -47,7 +52,6 @@
         owner: "root"
         group: "root"
         mode: "0644"
-  when: ceph_nfs_dynamic_exports
 
     - name: reload dbus configuration
       command: "killall -SIGHUP dbus-daemon"

roles/ceph-nfs/tasks/pre_requisite_non_container.yml

@@ -42,7 +42,9 @@
   when: item.create|bool
 
 - name: cephx related tasks
-  when: cephx
+  when:
+    - cephx
+    - groups.get(mon_group_name, []) | length > 0
   block:
     - name: copy bootstrap cephx keys
       copy:

roles/ceph-nfs/tasks/start_nfs.yml

@@ -1,4 +1,5 @@
 ---
+- block:
   - name: set_fact container_exec_cmd_nfs
     set_fact:
       container_exec_cmd_nfs: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
@@ -21,6 +22,7 @@
       - rados_index_exists.rc != 0
     delegate_to: "{{ groups[mon_group_name][0] }}"
     run_once: true
+  when: groups.get(mon_group_name, []) | length > 0
 
 - name: create /etc/ganesha
   file: