nfs: fix container setup and re-arrange files

Signed-off-by: Sébastien Han <seb@redhat.com>

group_vars/all.yml.sample

@@ -394,6 +394,10 @@ dummy:
 #handler_health_rgw_check_retries: 5
 #handler_health_rgw_check_delay: 10
 
+# NFS handler checks
+#handler_health_nfs_check_retries: 5
+#handler_health_nfs_check_delay: 10
+
 ###############
 # NFS-GANESHA #
 ###############

group_vars/rhcs.yml.sample

@@ -394,6 +394,10 @@ ceph_repository: rhcs
 #handler_health_rgw_check_retries: 5
 #handler_health_rgw_check_delay: 10
 
+# NFS handler checks
+#handler_health_nfs_check_retries: 5
+#handler_health_nfs_check_delay: 10
+
 ###############
 # NFS-GANESHA #
 ###############

roles/ceph-defaults/defaults/main.yml

@@ -386,6 +386,10 @@ handler_health_mds_check_delay: 10
 handler_health_rgw_check_retries: 5
 handler_health_rgw_check_delay: 10
 
+# NFS handler checks
+handler_health_nfs_check_retries: 5
+handler_health_nfs_check_delay: 10
+
 ###############
 # NFS-GANESHA #
 ###############

roles/ceph-defaults/templates/restart_nfs_daemon.sh.j2

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+RETRIES="{{ handler_health_nfs_check_retries }}"
+DELAY="{{ handler_health_nfs_check_delay }}"
+NFS_NAME="{{ ansible_hostname }}"
+PID=/var/run/ganesha.pid
+
+# First, restart the daemon
+{% if containerized_deployment -%}
+systemctl restart ceph-nfs@${NFS_NAME}
+COUNT=10
+# Wait and ensure the pid exists after restarting the daemon
+while [ $RETRIES -ne 0 ]; do
+  {{ docker_exec_cmd }} test -f $PID && exit 0
+  sleep $DELAY
+  let RETRIES=RETRIES-1
+done
+# If we reach this point, it means the pid is not present.
+echo "PID file ${PID} could not be found, which means Ganesha is not running."
+exit 1
+{% else %}
+systemctl restart nfs-ganesha
+{% endif %}

roles/ceph-nfs/tasks/create_rgw_nfs_user.yml

@@ -8,6 +8,7 @@
 - name: create rgw nfs user
   command: "{{ docker_exec_cmd_nfs | default('') }} radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name='RGW NFS User'"
   register: rgwuser
+  changed_when: false
   delegate_to: "{{ groups[mon_group_name][0] }}"
   when:
     - nfs_obj_gw

roles/ceph-nfs/tasks/docker/main.yml

@@ -1,3 +0,0 @@
----
-- name: include start_docker_nfs.yml
-  include: start_docker_nfs.yml

roles/ceph-nfs/tasks/docker/start_docker_nfs.yml

@@ -1,18 +0,0 @@
----
-- name: generate systemd unit file
-  become: true
-  template:
-    src: "{{ role_path }}/templates/ceph-nfs.service.j2"
-    dest: /etc/systemd/system/ceph-nfs@.service
-    owner: "root"
-    group: "root"
-    mode: "0644"
-
-- name: systemd start nfs container
-  systemd:
-    name: "ceph-nfs@{{ ansible_hostname }}.service"
-    state: started
-    enabled: yes
-    daemon_reload: yes
-  when:
-    - ceph_nfs_enable_service

roles/ceph-nfs/tasks/ganesha_selinux_fix.yml

@@ -0,0 +1,28 @@
+---
+- name: check if selinux is enabled
+  command: getenforce
+  register: selinuxstatus
+  changed_when: false
+  failed_when: false
+  always_run: true
+
+- name: install policycoreutils-python to get semanage
+  package:
+    name: policycoreutils-python
+    state: present
+  when:
+    - selinuxstatus.stdout != 'Disabled'
+
+- name: test if ganesha_t is already permissive
+  shell: |
+    semanage permissive -l | grep -soq ganesha_t
+  changed_when: false
+  failed_when: false
+  register: ganesha_t_permissive
+
+- name: run semanage permissive -a ganesha_t
+  command: semanage permissive -a ganesha_t
+  changed_when: false
+  when:
+    - selinuxstatus.stdout != 'Disabled'
+    - ganesha_t_permissive.rc != 0

roles/ceph-nfs/tasks/main.yml

@@ -1,18 +1,24 @@
 ---
-- name: include pre_requisite.yml
-  include: pre_requisite.yml
+- name: include pre_requisite_non_container.yml
+  include: pre_requisite_non_container.yml
   when:
     - not containerized_deployment
 
+- name: include pre_requisite_container.yml
+  include: pre_requisite_container.yml
+  when:
+    - containerized_deployment
+
 - name: include create_rgw_nfs_user.yml
   include: create_rgw_nfs_user.yml
 
-- name: include start_nfs.yml
-  include: start_nfs.yml
+# NOTE (leseb): workaround for issues with ganesha and librgw
+- name: include ganesha_selinux_fix.yml
+  include: ganesha_selinux_fix.yml
   when:
     - not containerized_deployment
+    - ansible_os_family == 'RedHat'
+    - ansible_distribution_version >= '7.4'
 
-- name: include docker/main.yml
-  include: docker/main.yml
-  when:
-    - containerized_deployment
+- name: include start_nfs.yml
+  include: start_nfs.yml

roles/ceph-nfs/tasks/pre_requisite_non_container.yml

@@ -1,4 +1,6 @@
 ---
+# NOTE (leseb): we use root:ceph for permissions since ganesha
+# does not have the right selinux context to read ceph directories.
 - name: create rados gateway and ganesha directories
   file:
     path: "{{ item }}"
@@ -11,8 +13,8 @@
     - /var/lib/ceph/radosgw
     - /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}
     - "{{ rbd_client_admin_socket_path }}"
-    - /var/lib/nfs/ganesha
-    - /var/run/ganesha
+    - /var/log/ceph
+    - /var/run/ceph/
   when:
     - nfs_obj_gw
 
@@ -51,7 +53,7 @@
 
 - name: change ownership on /var/log/ganesha
   file:
-    path: '/var/log/ganesha'
-    owner: 'root'
-    group: 'root'
-    mode: '0755'
+    path: /var/log/ganesha
+    owner: "root"
+    group: "root"
+    mode: "0755"

roles/ceph-nfs/tasks/start_nfs.yml

@@ -1,4 +1,12 @@
 ---
+- name: create /etc/ganesha
+  file:
+    path: /etc/ganesha
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
 - name: generate ganesha configuration file
   action: config_template
   args:
@@ -11,6 +19,27 @@
   notify:
     - restart ceph nfss
 
+- name: generate systemd unit file
+  become: true
+  template:
+    src: "{{ role_path }}/templates/ceph-nfs.service.j2"
+    dest: /etc/systemd/system/ceph-nfs@.service
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  when:
+    - containerized_deployment
+
+- name: systemd start nfs container
+  systemd:
+    name: "ceph-nfs@{{ ansible_hostname }}.service"
+    state: started
+    enabled: yes
+    daemon_reload: yes
+  when:
+    - ceph_nfs_enable_service
+    - containerized_deployment
+
 - name: start nfs gateway service
   service:
     name: nfs-ganesha
@@ -18,3 +47,4 @@
     enabled: yes
   when:
     - ceph_nfs_enable_service
+    - not containerized_deployment

roles/ceph-nfs/templates/ceph-nfs.service.j2

@@ -9,7 +9,9 @@ ExecStartPre=-/usr/bin/docker rm ceph-nfs-%i
 ExecStartPre=/usr/bin/mkdir -p /etc/ceph /etc/ganesha /var/lib/nfs/ganesha
 ExecStart=/usr/bin/docker run --rm --net=host \
   {% if not containerized_deployment_with_kv -%}
+  -v /var/lib/ceph:/var/lib/ceph \
   -v /etc/ceph:/etc/ceph \
+  -v /var/lib/ganesha:/var/lib/ganesha \
   -v /etc/ganesha:/etc/ganesha \
   {% else -%}
   -e KV_TYPE={{kv_type}} \
@@ -17,7 +19,7 @@ ExecStart=/usr/bin/docker run --rm --net=host \
   -e KV_PORT={{kv_port}} \
   {% endif -%}
   -v /etc/localtime:/etc/localtime:ro \
-   --privileged \
+  -e CLUSTER={{ cluster }} \
   -e CEPH_DAEMON=NFS \
   {{ ceph_nfs_docker_extra_env }} \
   --name=ceph-nfs-{{ ansible_hostname }} \