Merge pull request #932 from dang/nfs-rgw

NFS for FSAL_RGW

group_vars/nfss.sample

@@ -35,6 +35,28 @@ dummy:
 #ceph_nfs_pseudo_path: "/ceph"
 #ceph_nfs_protocols: "3,4"
 #ceph_nfs_access_type: "RW"
+#ceph_nfs_log_file: "/var/log/ganesha.log"
+
+####################
+# FSAL Ceph Config #
+####################
+#ceph_nfs_ceph_export_id: 20134
+#ceph_nfs_ceph_pseudo_path: "/ceph"
+#ceph_nfs_ceph_protocols: "3,4"
+#ceph_nfs_ceph_access_type: "RW"
+
+###################
+# FSAL RGW Config #
+###################
+#ceph_nfs_rgw_export_id: 20134
+#ceph_nfs_rgw_pseudo_path: "/ceph"
+#ceph_nfs_rgw_protocols: "3,4"
+#ceph_nfs_rgw_access_type: "RW"
+#ceph_nfs_rgw_user: "cephnfs"
+# Note: keys are optional and can be generated, but not on containerized, where
+# they must be configered.
+#ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY"
+#ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C"
 
 
 ###################

roles/ceph-common/tasks/installs/install_on_redhat.yml

@@ -227,18 +227,40 @@
     - rgw_group_name in group_names
     - ansible_pkg_mgr == "dnf"
 
-- name: install NFS gateway
+- name: install nfs ceph gateway
   yum:
     name: nfs-ganesha-ceph
-    state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}"
   when:
     - nfs_group_name in group_names
     - ansible_pkg_mgr == "yum"
+    - fsal_ceph
 
-- name: install NFS gateway
+- name: install nfs ceph gateway
   dnf:
     name: nfs-ganesha-ceph
-    state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}"
   when:
     - nfs_group_name in group_names
     - ansible_pkg_mgr == "dnf"
+    - fsal_ceph
+
+- name: install nfs rgw gateway
+  yum:
+    name: "{{ item }}"
+  with_items:
+    - nfs-ganesha-rgw
+    - ceph-radosgw
+  when:
+    - nfs_group_name in group_names
+    - ansible_pkg_mgr == "yum"
+    - fsal_rgw
+
+- name: install nfs rgw gateway
+  dnf:
+    name: "{{ item }}"
+  with_items:
+    - nfs-ganesha-rgw
+    - ceph-radosgw
+  when:
+    - nfs_group_name in group_names
+    - ansible_pkg_mgr == "dnf"
+    - fsal_rgw

roles/ceph-common/templates/ganesha.conf.j2

@@ -1,17 +1,18 @@
 #jinja2: trim_blocks: "true", lstrip_blocks: "true"
 # {{ ansible_managed }}
 
+{% if fsal_ceph %}
 EXPORT
 {
-	Export_ID={{ ceph_nfs_export_id }};
+	Export_ID={{ ceph_nfs_ceph_export_id }};
 
 	Path = "/";
 
-	Pseudo = {{ ceph_nfs_pseudo_path }};
+	Pseudo = {{ ceph_nfs_ceph_pseudo_path }};
 
-	Access_Type = {{ ceph_nfs_access_type }};
+	Access_Type = {{ ceph_nfs_ceph_access_type }};
 
-	NFS_Protocols = {{ ceph_nfs_protocols }};
+	NFS_Protocols = {{ ceph_nfs_ceph_protocols }};
 
 	Transport_Protocols = TCP;
 
@@ -21,3 +22,37 @@ EXPORT
 		Name = CEPH;
 	}
 }
+{% endif %}
+{% if fsal_rgw %}
+EXPORT
+{
+	Export_ID={{ ceph_nfs_rgw_export_id }};
+
+	Path = "/";
+
+	Pseudo = {{ ceph_nfs_rgw_pseudo_path }};
+
+	Access_Type = {{ ceph_nfs_rgw_access_type }};
+
+	NFS_Protocols = {{ ceph_nfs_rgw_protocols }};
+
+	Transport_Protocols = TCP;
+
+	Sectype = sys,krb5,krb5i,krb5p;
+
+	FSAL {
+		Name = RGW;
+		User_Id = "{{ ceph_nfs_rgw_user }}";
+		Access_Key_Id ="{{ ceph_nfs_rgw_access_key }}";
+		Secret_Access_Key = "{{ ceph_nfs_rgw_secret_key }}";
+	}
+}
+{% endif %}
+
+LOG {
+        Facility {
+                name = FILE;
+                destination = "{{ ceph_nfs_log_file }}";
+                enable = active;
+        }
+}

roles/ceph-nfs/defaults/main.yml

@@ -27,6 +27,28 @@ ceph_nfs_export_id: 20134
 ceph_nfs_pseudo_path: "/ceph"
 ceph_nfs_protocols: "3,4"
 ceph_nfs_access_type: "RW"
+ceph_nfs_log_file: "/var/log/ganesha.log"
+
+####################
+# FSAL Ceph Config #
+####################
+ceph_nfs_ceph_export_id: 20134
+ceph_nfs_ceph_pseudo_path: "/ceph"
+ceph_nfs_ceph_protocols: "3,4"
+ceph_nfs_ceph_access_type: "RW"
+
+###################
+# FSAL RGW Config #
+###################
+ceph_nfs_rgw_export_id: 20134
+ceph_nfs_rgw_pseudo_path: "/ceph"
+ceph_nfs_rgw_protocols: "3,4"
+ceph_nfs_rgw_access_type: "RW"
+ceph_nfs_rgw_user: "cephnfs"
+# Note: keys are optional and can be generated, but not on containerized, where
+# they must be configered.
+#ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY"
+#ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C"
 
 
 ###################

roles/ceph-nfs/tasks/docker/copy_configs.yml

@@ -1,10 +1,28 @@
 ---
-- name: push ceph files to the ansible server
-  fetch:
-    src: "{{ item.0 }}"
-    dest: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
-    flat: yes
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
+      - /etc/ceph/ceph.conf
+      - /etc/ceph/ceph.client.admin.keyring
+      - /var/lib/ceph/radosgw/keyring
+
+- name: stat for config and keys
+  local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
+  with_items: ceph_config_keys
+  changed_when: false
+  become: false
+  failed_when: false
+  register: statconfig
+
+- name: try to fetch config and keys
+  copy:
+    src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
+    dest: "{{ item.0 }}"
+    owner: "64045"
+    group: "64045"
+    mode: 0644
+  changed_when: false
   with_together:
     - ceph_config_keys
     - statconfig.results
-  when: item.1.stat.exists == false
+  when: item.1.stat.exists == true

roles/ceph-nfs/tasks/docker/create_configs.yml

@@ -7,6 +7,21 @@
     group: root
     mode: 0644
 
+- name: create the nfs rgw user
+  docker:
+    image: "{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}:{{ ceph_rgw_docker_image_tag }}"
+    name: ceph-{{ ansible_hostname }}-rgw-user
+    expose: "{{ ceph_rgw_civetweb_port }}"
+    ports: "{{ ceph_rgw_civetweb_port }}:{{ ceph_rgw_civetweb_port }}"
+    state: running
+    env: "CEPH_DAEMON=RGW_USER,RGW_USERNAME={{ ceph_nfs_rgw_user }},RGW_USER_ACCESS_KEY={{ ceph_nfs_rgw_access_key }},RGW_USER_SECRET_KEY={{ ceph_nfs_rgw_secret_key }}"
+    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
+  when: fsal_rgw
+
+- name: get user create output
+  command: docker logs ceph-{{ ansible_hostname }}-rgw-user
+  register: rgwuser
+
 - name: generate ganesha configuration file
   action: config_template
   args:

roles/ceph-nfs/tasks/docker/dirs_permissions.yml

@@ -22,6 +22,7 @@
   with_items:
     - /etc/ceph/
     - /var/lib/ceph/
+    - /var/lib/ceph/radosgw
   when: not after_hammer
 
 - name: create bootstrap directories (after hammer)
@@ -34,6 +35,7 @@
   with_items:
     - /etc/ceph/
     - /var/lib/ceph/
+    - /var/lib/ceph/radosgw
   when: after_hammer
 
 - name: create ganesha directories

roles/ceph-nfs/tasks/docker/fetch_configs.yml

@@ -1,13 +1,12 @@
 ---
-- name: set config and keys paths
+- name: set config paths
   set_fact:
-    ceph_config_keys:
-      - /etc/ceph/ceph.conf
+    nfs_config_keys:
       - /etc/ganesha/ganesha.conf
 
 - name: stat for config and keys
   local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
-  with_items: ceph_config_keys
+  with_items: nfs_config_keys
   changed_when: false
   become: false
   failed_when: false
@@ -22,6 +21,16 @@
     mode: 0644
   changed_when: false
   with_together:
-    - ceph_config_keys
+    - nfs_config_keys
     - statconfig.results
   when: item.1.stat.exists == true
+
+- name: push ganesha files to the ansible server
+  fetch:
+    src: "{{ item.0 }}"
+    dest: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
+    flat: yes
+  with_together:
+    - nfs_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == false

roles/ceph-nfs/tasks/docker/main.yml

@@ -45,12 +45,19 @@
 
 - include: dirs_permissions.yml
 
-# let the first ganesha create configs and keyrings
+# Copy Ceph configs to host
+- include: copy_configs.yml
+
+- include: selinux.yml
+  when: ansible_os_family == 'RedHat'
+
+# let the first ganesha create configs and users
 - include: create_configs.yml
   when:
     inventory_hostname == groups.nfss[0] and
     mon_containerized_default_ceph_conf_with_kv
 
+# Copy Ganesha configs to host
 - include: fetch_configs.yml
   when: not mon_containerized_deployment_with_kv
 
@@ -59,5 +66,3 @@
 
 - include: start_docker_nfs.yml
 
-- include: copy_configs.yml
-  when: not mon_containerized_deployment_with_kv

roles/ceph-nfs/tasks/pre_requisite.yml

@@ -1,5 +1,5 @@
 ---
-- name: create NFS gateway directories
+- name: create nfs gateway directories
   file:
     path: "{{ item }}"
     state: directory
@@ -10,7 +10,33 @@
     - /var/lib/nfs/ganesha
     - /var/run/ganesha
 
-- name: start NFS gateway service
+- name: create rgw nfs user
+  command: radosgw-admin user create --uid={{ ceph_nfs_rgw_user }} --display-name="RGW NFS User"
+  register: rgwuser
+  when: fsal_rgw
+
+- name: set access key
+  set_fact:
+    ceph_nfs_rgw_access_key: "{{ (rgwuser.stdout | from_json)['keys'][0]['access_key'] }}"
+  when: fsal_rgw
+
+- name: set secret key
+  set_fact:
+    ceph_nfs_rgw_secret_key: "{{(rgwuser.stdout | from_json)['keys'][0]['secret_key']}}"
+  when: fsal_rgw
+
+- name: generate ganesha configuration file
+  action: config_template
+  args:
+    src: "{{ playbook_dir }}/roles/ceph-common/templates/ganesha.conf.j2"
+    dest: /etc/ganesha/ganesha.conf
+    owner: "root"
+    group: "root"
+    mode: "0644"
+    config_overrides: "{{ ganesha_conf_overrides }}"
+    config_type: ini
+
+- name: start nfs gateway service
   service:
     name: nfs-ganesha
     state: started

roles/ceph-nfs/templates/ceph-nfs.service.j2

@@ -7,7 +7,7 @@ After=docker.service
 EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker rm %i
 ExecStartPre=/usr/bin/mkdir -p /etc/ceph /etc/ganesha /var/lib/nfs/ganesha
-ExecStart=/usr/bin/docker run --rm --name %i --net=host \
+ExecStart=/usr/bin/docker run --rm --net=host \
    {% if not mon_containerized_deployment_with_kv -%}
    -v /etc/ceph:/etc/ceph \
    -v /etc/ganesha:/etc/ganesha \
@@ -18,8 +18,7 @@ ExecStart=/usr/bin/docker run --rm --name %i --net=host \
    -v /etc/localtime:/etc/localtime:ro \
    --privileged \
    -e CEPH_DAEMON=NFS \
-   -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \
-   --name={{ ansible_hostname }} \
+   --name=nfs-{{ ansible_hostname }} \
     {{ ceph_nfs_docker_username }}/{{ ceph_nfs_docker_imagename }}:{{ ceph_nfs_docker_image_tag }}
 ExecStopPost=-/usr/bin/docker stop %i
 Restart=always

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -31,8 +31,8 @@
     -v /etc/localtime:/etc/localtime:ro \
     -e "OSD_DEVICE={{ item.0 }}" \
     -e "{{ ceph_osd_docker_prepare_env }}" \
-    "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}" \
-    OSD_CEPH_DISK_PREPARE
+    -e CEPH_DAEMON=osd_ceph_disk_prepare \
+    "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}"
   with_together:
     - ceph_osd_docker_devices
     - osd_prepared.results

roles/ceph-rgw/tasks/docker/copy_configs.yml

@@ -0,0 +1,36 @@
+---
+- name: set config and keys paths
+  set_fact:
+    rgw_config_keys:
+      - "/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring"
+  when: fsal_rgw
+
+- name: wait for rgw keyring
+  wait_for: path="/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring"
+  when:
+    - fsal_rgw
+    - inventory_hostname == groups.rgws[0]
+
+- name: stat for config and keys
+  local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
+  with_items: rgw_config_keys
+  changed_when: false
+  become: false
+  failed_when: false
+  register: statconfig
+  when:
+    - fsal_rgw
+    - inventory_hostname == groups.rgws[0]
+
+- name: push ceph files to the ansible server
+  fetch:
+    src: "{{ item.0 }}"
+    dest: "{{ fetch_directory }}/docker_mon_files/var/lib/ceph/radosgw/keyring"
+    flat: yes
+  with_together:
+    - rgw_config_keys
+    - statconfig.results
+  when:
+    - item.1.stat.exists == false
+    - fsal_rgw
+    - inventory_hostname == groups.rgws[0]

roles/ceph-rgw/tasks/docker/main.yml

@@ -49,3 +49,5 @@
   when: ansible_os_family == 'RedHat'
 
 - include: start_docker_rgw.yml
+
+- include: copy_configs.yml