nfs: support internal Ganesha with external ceph cluster

This commits allows to deploy an internal ganesha with an external ceph
cluster.

This requires to define `external_cluster_mon_ips` with a comma
separated list of external monitors.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1710358

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
(cherry picked from commit 6a6785b719)

group_vars/all.yml.sample

@@ -474,7 +474,7 @@ dummy:
 # Set this to true to enable File access via NFS.  Requires an MDS role.
 #nfs_file_gw: false
 # Set this to true to enable Object access via NFS. Requires an RGW role.
-#nfs_obj_gw: true
+#nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}"
 
 
 #############

group_vars/clients.yml.sample

@@ -57,3 +57,4 @@ dummy:
 #  - { name: client.test, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test" },  mode: "{{ ceph_keyring_permissions }}" }
 #  - { name: client.test2, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test2" },  mode: "{{ ceph_keyring_permissions }}" }
 
+#ceph_nfs_ceph_user: { name: client.rgw.nfs0, key: 'SECRET==', caps: { mon: "allow rw", osd: "allow rwx" }, mode: "{{ ceph_keyring_permissions }}" }

group_vars/rhcs.yml.sample

@@ -474,7 +474,7 @@ ceph_rhcs_version: 4
 # Set this to true to enable File access via NFS.  Requires an MDS role.
 #nfs_file_gw: false
 # Set this to true to enable Object access via NFS. Requires an RGW role.
-#nfs_obj_gw: true
+#nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}"
 
 
 #############

roles/ceph-client/defaults/main.yml

@@ -48,3 +48,5 @@ pools:
 keys:
   - { name: client.test, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test" },  mode: "{{ ceph_keyring_permissions }}" }
   - { name: client.test2, caps: { mon: "allow r", osd: "allow class-read object_prefix rbd_children, allow rwx pool=test2" },  mode: "{{ ceph_keyring_permissions }}" }
+
+#ceph_nfs_ceph_user: { name: client.rgw.nfs0, key: 'SECRET==', caps: { mon: "allow rw", osd: "allow rwx" }, mode: "{{ ceph_keyring_permissions }}" }

roles/ceph-client/tasks/create_users_keys.yml

@@ -45,7 +45,9 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
     CEPH_CONTAINER_BINARY: "{{ container_binary }}"
-  with_items: "{{ keys }}"
+  with_items:
+    - "{{ keys }}"
+    - "{{ ceph_nfs_ceph_user | default([]) }}"
   delegate_to: "{{ delegated_node }}"
   when:
     - cephx

roles/ceph-config/templates/ceph.conf.j2

@@ -56,7 +56,7 @@ mon host = {% if nb_mon > 0 %}
 {%- endif %}
 {%- if not loop.last -%},{%- endif %}
 {%- endfor %}
-{% elif nb_mon == 0 and inventory_hostname in groups.get(client_group_name, []) %}
+{% elif nb_mon == 0 %}
 {{ external_cluster_mon_ips }}
 {% endif %}
 

roles/ceph-defaults/defaults/main.yml

@@ -466,7 +466,7 @@ handler_health_mgr_check_delay: 10
 # Set this to true to enable File access via NFS.  Requires an MDS role.
 nfs_file_gw: false
 # Set this to true to enable Object access via NFS. Requires an RGW role.
-nfs_obj_gw: true
+nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}"
 
 
 #############

roles/ceph-nfs/tasks/common.yml

@@ -12,3 +12,4 @@
   when:
     - cephx
     - item.copy_key|bool
+    - groups.get(mon_group_name, []) | length > 0

roles/ceph-nfs/tasks/main.yml

@@ -17,6 +17,7 @@
 
 - name: include create_rgw_nfs_user.yml
   import_tasks: create_rgw_nfs_user.yml
+  when: groups.get(mon_group_name, []) | length > 0
 
 # NOTE (leseb): workaround for issues with ganesha and librgw
 - name: include ganesha_selinux_fix.yml
@@ -25,5 +26,17 @@
     - not containerized_deployment
     - ansible_os_family == 'RedHat'
 
+- name: copy rgw keyring when deploying internal ganesha with external ceph cluster
+  copy:
+    src: "/etc/ceph/{{ cluster }}.{{ ceph_nfs_ceph_user.name }}.keyring"
+    dest: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring"
+    mode: '0600'
+    owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+    group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+    remote_src: true
+  when:
+    - groups.get(mon_group_name, []) | length == 0
+    - ceph_nfs_ceph_user is defined
+
 - name: include start_nfs.yml
   import_tasks: start_nfs.yml

roles/ceph-nfs/tasks/pre_requisite_container.yml

@@ -1,54 +1,58 @@
 ---
-- name: set_fact admin_keyring
-  set_fact:
-    admin_keyring:
-      - "/etc/ceph/{{ cluster }}.client.admin.keyring"
-  when: copy_admin_key
+- name: keyring related tasks
+  block:
+    - name: set_fact admin_keyring
+      set_fact:
+        admin_keyring:
+          - "/etc/ceph/{{ cluster }}.client.admin.keyring"
+      when: copy_admin_key
 
-- name: set_fact ceph_config_keys
-  set_fact:
-    ceph_config_keys:
-      - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring
+    - name: set_fact ceph_config_keys
+      set_fact:
+        ceph_config_keys:
+          - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring
 
-- name: merge ceph_config_keys and admin_keyring
-  set_fact:
-    ceph_config_keys: "{{ ceph_config_keys + admin_keyring }}"
-  when: copy_admin_key
+    - name: merge ceph_config_keys and admin_keyring
+      set_fact:
+        ceph_config_keys: "{{ ceph_config_keys + admin_keyring }}"
+      when: copy_admin_key
 
-- name: stat for config and keys
-  stat:
-    path: "{{ fetch_directory }}/{{ fsid }}/{{ item }}"
-  delegate_to: localhost
-  with_items: "{{ ceph_config_keys }}"
-  changed_when: false
-  become: false
-  failed_when: false
-  check_mode: no
-  register: statconfig
+    - name: stat for config and keys
+      stat:
+        path: "{{ fetch_directory }}/{{ fsid }}/{{ item }}"
+      delegate_to: localhost
+      with_items: "{{ ceph_config_keys }}"
+      changed_when: false
+      become: false
+      failed_when: false
+      check_mode: no
+      register: statconfig
 
-- name: try to fetch config and keys
-  copy:
-    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
-    dest: "{{ item.0 }}"
-    owner: "64045"
-    group: "64045"
-    mode: 0644
-  changed_when: false
-  with_together:
-    - "{{ ceph_config_keys }}"
-    - "{{ statconfig.results }}"
-  when: item.1.stat.exists
+    - name: try to fetch config and keys
+      copy:
+        src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+        dest: "{{ item.0 }}"
+        owner: "64045"
+        group: "64045"
+        mode: 0644
+      changed_when: false
+      with_together:
+        - "{{ ceph_config_keys }}"
+        - "{{ statconfig.results }}"
+      when: item.1.stat.exists
+  when: groups.get(mon_group_name, []) | length > 0
 
-- name: create dbus service file
-  become: true
-  copy:
-    src: "org.ganesha.nfsd.conf"
-    dest: /etc/dbus-1/system.d/org.ganesha.nfsd.conf
-    owner: "root"
-    group: "root"
-    mode: "0644"
-  when: ceph_nfs_dynamic_exports
-
-- name: reload dbus configuration
-  command: "killall -SIGHUP dbus-daemon"
+- name: dbus related tasks
+  block:
+    - name: create dbus service file
+      become: true
+      copy:
+        src: "org.ganesha.nfsd.conf"
+        dest: /etc/dbus-1/system.d/org.ganesha.nfsd.conf
+        owner: "root"
+        group: "root"
+        mode: "0644"
+
+    - name: reload dbus configuration
+      command: "killall -SIGHUP dbus-daemon"
   when: ceph_nfs_dynamic_exports

roles/ceph-nfs/tasks/pre_requisite_non_container.yml

@@ -42,7 +42,9 @@
   when: item.create|bool
 
 - name: cephx related tasks
-  when: cephx
+  when:
+    - cephx
+    - groups.get(mon_group_name, []) | length > 0
   block:
     - name: copy bootstrap cephx keys
       copy:

roles/ceph-nfs/tasks/start_nfs.yml

@@ -1,26 +1,28 @@
 ---
-- name: set_fact container_exec_cmd_nfs
-  set_fact:
-    container_exec_cmd_nfs: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
-  when: containerized_deployment
+- block:
+  - name: set_fact container_exec_cmd_nfs
+    set_fact:
+      container_exec_cmd_nfs: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
+    when: containerized_deployment
 
-- name: check if rados index object exists
-  shell: "{{ container_exec_cmd_nfs | default('') }} rados -p {{ cephfs_data }} --cluster {{ cluster }} ls|grep {{ ceph_nfs_rados_export_index }}"
-  changed_when: false
-  failed_when: false
-  register: rados_index_exists
-  check_mode: no
-  when: ceph_nfs_rados_backend
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  run_once: true
+  - name: check if rados index object exists
+    shell: "{{ container_exec_cmd_nfs | default('') }} rados -p {{ cephfs_data }} --cluster {{ cluster }} ls|grep {{ ceph_nfs_rados_export_index }}"
+    changed_when: false
+    failed_when: false
+    register: rados_index_exists
+    check_mode: no
+    when: ceph_nfs_rados_backend
+    delegate_to: "{{ groups[mon_group_name][0] }}"
+    run_once: true
 
-- name: create an empty rados index object
-  command: "{{ container_exec_cmd_nfs | default('') }} rados -p {{ cephfs_data }} --cluster {{ cluster }} put {{ ceph_nfs_rados_export_index }} /dev/null"
-  when:
-    - ceph_nfs_rados_backend
-    - rados_index_exists.rc != 0
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  run_once: true
+  - name: create an empty rados index object
+    command: "{{ container_exec_cmd_nfs | default('') }} rados -p {{ cephfs_data }} --cluster {{ cluster }} put {{ ceph_nfs_rados_export_index }} /dev/null"
+    when:
+      - ceph_nfs_rados_backend
+      - rados_index_exists.rc != 0
+    delegate_to: "{{ groups[mon_group_name][0] }}"
+    run_once: true
+  when: groups.get(mon_group_name, []) | length > 0
 
 - name: create /etc/ganesha
   file: