From 341c9e077b98b23a38db69d2704ddffb9c752ee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 26 Sep 2017 23:16:43 +0200 Subject: [PATCH] nfs: fix container setup and re-arrange files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sébastien Han --- group_vars/all.yml.sample | 4 +++ group_vars/rhcs.yml.sample | 4 +++ roles/ceph-defaults/defaults/main.yml | 4 +++ .../templates/restart_nfs_daemon.sh.j2 | 23 ++++++++++++++ roles/ceph-nfs/tasks/create_rgw_nfs_user.yml | 1 + roles/ceph-nfs/tasks/docker/main.yml | 3 -- .../tasks/docker/start_docker_nfs.yml | 18 ----------- roles/ceph-nfs/tasks/ganesha_selinux_fix.yml | 28 +++++++++++++++++ roles/ceph-nfs/tasks/main.yml | 22 +++++++++----- ...onfigs.yml => pre_requisite_container.yml} | 0 ...te.yml => pre_requisite_non_container.yml} | 14 +++++---- roles/ceph-nfs/tasks/start_nfs.yml | 30 +++++++++++++++++++ roles/ceph-nfs/templates/ceph-nfs.service.j2 | 30 ++++++++++--------- 13 files changed, 132 insertions(+), 49 deletions(-) create mode 100644 roles/ceph-defaults/templates/restart_nfs_daemon.sh.j2 delete mode 100644 roles/ceph-nfs/tasks/docker/main.yml delete mode 100644 roles/ceph-nfs/tasks/docker/start_docker_nfs.yml create mode 100644 roles/ceph-nfs/tasks/ganesha_selinux_fix.yml rename roles/ceph-nfs/tasks/{docker/copy_configs.yml => pre_requisite_container.yml} (100%) rename roles/ceph-nfs/tasks/{pre_requisite.yml => pre_requisite_non_container.yml} (86%) diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample index 9c10d5adf..e2d54f78b 100644 --- a/group_vars/all.yml.sample +++ b/group_vars/all.yml.sample @@ -394,6 +394,10 @@ dummy: #handler_health_rgw_check_retries: 5 #handler_health_rgw_check_delay: 10 +# NFS handler checks +#handler_health_nfs_check_retries: 5 +#handler_health_nfs_check_delay: 10 + ############### # NFS-GANESHA # ############### diff --git a/group_vars/rhcs.yml.sample b/group_vars/rhcs.yml.sample index 5e9f926ea..112efa41e 100644 --- a/group_vars/rhcs.yml.sample +++ b/group_vars/rhcs.yml.sample @@ -394,6 +394,10 @@ ceph_repository: rhcs #handler_health_rgw_check_retries: 5 #handler_health_rgw_check_delay: 10 +# NFS handler checks +#handler_health_nfs_check_retries: 5 +#handler_health_nfs_check_delay: 10 + ############### # NFS-GANESHA # ############### diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml index f8786be1c..cfdbbbdca 100644 --- a/roles/ceph-defaults/defaults/main.yml +++ b/roles/ceph-defaults/defaults/main.yml @@ -386,6 +386,10 @@ handler_health_mds_check_delay: 10 handler_health_rgw_check_retries: 5 handler_health_rgw_check_delay: 10 +# NFS handler checks +handler_health_nfs_check_retries: 5 +handler_health_nfs_check_delay: 10 + ############### # NFS-GANESHA # ############### diff --git a/roles/ceph-defaults/templates/restart_nfs_daemon.sh.j2 b/roles/ceph-defaults/templates/restart_nfs_daemon.sh.j2 new file mode 100644 index 000000000..cbc78e989 --- /dev/null +++ b/roles/ceph-defaults/templates/restart_nfs_daemon.sh.j2 @@ -0,0 +1,23 @@ +#!/bin/bash + +RETRIES="{{ handler_health_nfs_check_retries }}" +DELAY="{{ handler_health_nfs_check_delay }}" +NFS_NAME="{{ ansible_hostname }}" +PID=/var/run/ganesha.pid + +# First, restart the daemon +{% if containerized_deployment -%} +systemctl restart ceph-nfs@${NFS_NAME} +COUNT=10 +# Wait and ensure the pid exists after restarting the daemon +while [ $RETRIES -ne 0 ]; do + {{ docker_exec_cmd }} test -f $PID && exit 0 + sleep $DELAY + let RETRIES=RETRIES-1 +done +# If we reach this point, it means the pid is not present. +echo "PID file ${PID} could not be found, which means Ganesha is not running." +exit 1 +{% else %} +systemctl restart nfs-ganesha +{% endif %} diff --git a/roles/ceph-nfs/tasks/create_rgw_nfs_user.yml b/roles/ceph-nfs/tasks/create_rgw_nfs_user.yml index 9eb82832e..1f51fbf1a 100644 --- a/roles/ceph-nfs/tasks/create_rgw_nfs_user.yml +++ b/roles/ceph-nfs/tasks/create_rgw_nfs_user.yml @@ -8,6 +8,7 @@ - name: create rgw nfs user command: "{{ docker_exec_cmd_nfs | default('') }} radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name='RGW NFS User'" register: rgwuser + changed_when: false delegate_to: "{{ groups[mon_group_name][0] }}" when: - nfs_obj_gw diff --git a/roles/ceph-nfs/tasks/docker/main.yml b/roles/ceph-nfs/tasks/docker/main.yml deleted file mode 100644 index f05ce4ae9..000000000 --- a/roles/ceph-nfs/tasks/docker/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- name: include start_docker_nfs.yml - include: start_docker_nfs.yml diff --git a/roles/ceph-nfs/tasks/docker/start_docker_nfs.yml b/roles/ceph-nfs/tasks/docker/start_docker_nfs.yml deleted file mode 100644 index 45bc18eb2..000000000 --- a/roles/ceph-nfs/tasks/docker/start_docker_nfs.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: generate systemd unit file - become: true - template: - src: "{{ role_path }}/templates/ceph-nfs.service.j2" - dest: /etc/systemd/system/ceph-nfs@.service - owner: "root" - group: "root" - mode: "0644" - -- name: systemd start nfs container - systemd: - name: "ceph-nfs@{{ ansible_hostname }}.service" - state: started - enabled: yes - daemon_reload: yes - when: - - ceph_nfs_enable_service diff --git a/roles/ceph-nfs/tasks/ganesha_selinux_fix.yml b/roles/ceph-nfs/tasks/ganesha_selinux_fix.yml new file mode 100644 index 000000000..0aa3c66ed --- /dev/null +++ b/roles/ceph-nfs/tasks/ganesha_selinux_fix.yml @@ -0,0 +1,28 @@ +--- +- name: check if selinux is enabled + command: getenforce + register: selinuxstatus + changed_when: false + failed_when: false + always_run: true + +- name: install policycoreutils-python to get semanage + package: + name: policycoreutils-python + state: present + when: + - selinuxstatus.stdout != 'Disabled' + +- name: test if ganesha_t is already permissive + shell: | + semanage permissive -l | grep -soq ganesha_t + changed_when: false + failed_when: false + register: ganesha_t_permissive + +- name: run semanage permissive -a ganesha_t + command: semanage permissive -a ganesha_t + changed_when: false + when: + - selinuxstatus.stdout != 'Disabled' + - ganesha_t_permissive.rc != 0 diff --git a/roles/ceph-nfs/tasks/main.yml b/roles/ceph-nfs/tasks/main.yml index fe70f3c40..435628ed3 100644 --- a/roles/ceph-nfs/tasks/main.yml +++ b/roles/ceph-nfs/tasks/main.yml @@ -1,18 +1,24 @@ --- -- name: include pre_requisite.yml - include: pre_requisite.yml +- name: include pre_requisite_non_container.yml + include: pre_requisite_non_container.yml when: - not containerized_deployment +- name: include pre_requisite_container.yml + include: pre_requisite_container.yml + when: + - containerized_deployment + - name: include create_rgw_nfs_user.yml include: create_rgw_nfs_user.yml -- name: include start_nfs.yml - include: start_nfs.yml +# NOTE (leseb): workaround for issues with ganesha and librgw +- name: include ganesha_selinux_fix.yml + include: ganesha_selinux_fix.yml when: - not containerized_deployment + - ansible_os_family == 'RedHat' + - ansible_distribution_version >= '7.4' -- name: include docker/main.yml - include: docker/main.yml - when: - - containerized_deployment +- name: include start_nfs.yml + include: start_nfs.yml diff --git a/roles/ceph-nfs/tasks/docker/copy_configs.yml b/roles/ceph-nfs/tasks/pre_requisite_container.yml similarity index 100% rename from roles/ceph-nfs/tasks/docker/copy_configs.yml rename to roles/ceph-nfs/tasks/pre_requisite_container.yml diff --git a/roles/ceph-nfs/tasks/pre_requisite.yml b/roles/ceph-nfs/tasks/pre_requisite_non_container.yml similarity index 86% rename from roles/ceph-nfs/tasks/pre_requisite.yml rename to roles/ceph-nfs/tasks/pre_requisite_non_container.yml index afe1c7094..0ab6bd22b 100644 --- a/roles/ceph-nfs/tasks/pre_requisite.yml +++ b/roles/ceph-nfs/tasks/pre_requisite_non_container.yml @@ -1,4 +1,6 @@ --- +# NOTE (leseb): we use root:ceph for permissions since ganesha +# does not have the right selinux context to read ceph directories. - name: create rados gateway and ganesha directories file: path: "{{ item }}" @@ -11,8 +13,8 @@ - /var/lib/ceph/radosgw - /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }} - "{{ rbd_client_admin_socket_path }}" - - /var/lib/nfs/ganesha - - /var/run/ganesha + - /var/log/ceph + - /var/run/ceph/ when: - nfs_obj_gw @@ -51,7 +53,7 @@ - name: change ownership on /var/log/ganesha file: - path: '/var/log/ganesha' - owner: 'root' - group: 'root' - mode: '0755' + path: /var/log/ganesha + owner: "root" + group: "root" + mode: "0755" diff --git a/roles/ceph-nfs/tasks/start_nfs.yml b/roles/ceph-nfs/tasks/start_nfs.yml index 02513149d..da5e4c3d7 100644 --- a/roles/ceph-nfs/tasks/start_nfs.yml +++ b/roles/ceph-nfs/tasks/start_nfs.yml @@ -1,4 +1,12 @@ --- +- name: create /etc/ganesha + file: + path: /etc/ganesha + state: directory + owner: root + group: root + mode: "0755" + - name: generate ganesha configuration file action: config_template args: @@ -11,6 +19,27 @@ notify: - restart ceph nfss +- name: generate systemd unit file + become: true + template: + src: "{{ role_path }}/templates/ceph-nfs.service.j2" + dest: /etc/systemd/system/ceph-nfs@.service + owner: "root" + group: "root" + mode: "0644" + when: + - containerized_deployment + +- name: systemd start nfs container + systemd: + name: "ceph-nfs@{{ ansible_hostname }}.service" + state: started + enabled: yes + daemon_reload: yes + when: + - ceph_nfs_enable_service + - containerized_deployment + - name: start nfs gateway service service: name: nfs-ganesha @@ -18,3 +47,4 @@ enabled: yes when: - ceph_nfs_enable_service + - not containerized_deployment diff --git a/roles/ceph-nfs/templates/ceph-nfs.service.j2 b/roles/ceph-nfs/templates/ceph-nfs.service.j2 index 1b0834ab1..fadfc3499 100644 --- a/roles/ceph-nfs/templates/ceph-nfs.service.j2 +++ b/roles/ceph-nfs/templates/ceph-nfs.service.j2 @@ -8,20 +8,22 @@ EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/docker rm ceph-nfs-%i ExecStartPre=/usr/bin/mkdir -p /etc/ceph /etc/ganesha /var/lib/nfs/ganesha ExecStart=/usr/bin/docker run --rm --net=host \ - {% if not containerized_deployment_with_kv -%} - -v /etc/ceph:/etc/ceph \ - -v /etc/ganesha:/etc/ganesha \ - {% else -%} - -e KV_TYPE={{kv_type}} \ - -e KV_IP={{kv_endpoint}}\ - -e KV_PORT={{kv_port}} \ - {% endif -%} - -v /etc/localtime:/etc/localtime:ro \ - --privileged \ - -e CEPH_DAEMON=NFS \ - {{ ceph_nfs_docker_extra_env }} \ - --name=ceph-nfs-{{ ansible_hostname }} \ - {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} + {% if not containerized_deployment_with_kv -%} + -v /var/lib/ceph:/var/lib/ceph \ + -v /etc/ceph:/etc/ceph \ + -v /var/lib/ganesha:/var/lib/ganesha \ + -v /etc/ganesha:/etc/ganesha \ + {% else -%} + -e KV_TYPE={{kv_type}} \ + -e KV_IP={{kv_endpoint}}\ + -e KV_PORT={{kv_port}} \ + {% endif -%} + -v /etc/localtime:/etc/localtime:ro \ + -e CLUSTER={{ cluster }} \ + -e CEPH_DAEMON=NFS \ + {{ ceph_nfs_docker_extra_env }} \ + --name=ceph-nfs-{{ ansible_hostname }} \ + {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} ExecStopPost=-/usr/bin/docker stop ceph-nfs-%i Restart=always RestartSec=10s