Merge pull request #1880 from ceph/wip-rgw-nfs

nfs: configure RGW FSAL to start up correctly

group_vars/nfss.yml.sample

@@ -35,7 +35,7 @@ dummy:
 #ceph_nfs_pseudo_path: "/cephfile"
 #ceph_nfs_protocols: "3,4"
 #ceph_nfs_access_type: "RW"
-#ceph_nfs_log_file: "/var/log/ganesha.log"
+#ceph_nfs_log_file: "/var/log/ganesha/ganesha.log"
 
 ####################
 # FSAL Ceph Config #
@@ -57,6 +57,7 @@ dummy:
 # they must be configered.
 #ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY"
 #ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C"
+#rgw_client_name: client.rgw.{{ ansible_hostname }}
 
 ###################
 # CONFIG OVERRIDE #
@@ -74,6 +75,7 @@ dummy:
 #
 #ganesha_ceph_export_overrides:
 #ganesha_rgw_export_overrides:
+#ganesha_rgw_section_overrides:
 #ganesha_log_overrides:
 #ganesha_conf_overrides: |
 #    CACHEINODE {

roles/ceph-config/templates/ceph.conf.j2

@@ -178,6 +178,42 @@ rgw data = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ hostvars[host]['ansible_ho
 {% endif %}
 {% endif %}
 
+{% if groups[nfs_group_name] is defined %}
+{% if nfs_group_name in group_names %}
+{% for host in groups[nfs_group_name] %}
+{% if nfs_obj_gw %}
+{% if hostvars[host]['ansible_hostname'] is defined %}
+[client.rgw.{{ hostvars[host]['ansible_hostname'] }}]
+host = {{ hostvars[host]['ansible_hostname'] }}
+keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ hostvars[host]['ansible_hostname'] }}/keyring
+log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_hostname'] }}.log
+rgw data = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ hostvars[host]['ansible_hostname'] }}
+{% if radosgw_address_block | length > 0 %}
+    {% if ip_version == 'ipv4' -%}
+      rgw frontends = civetweb port={{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {%- elif ip_version == 'ipv6' -%}
+      rgw frontends = civetweb port=[{{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {%- endif %}
+{% elif hostvars[host]['radosgw_address'] is defined and hostvars[host]['radosgw_address'] != '0.0.0.0' -%}
+    {% if ip_version == 'ipv4' -%}
+      rgw frontends = civetweb port={{ hostvars[host]['radosgw_address'] }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {%- elif ip_version == 'ipv6' -%}
+      rgw frontends = civetweb port=[{{ hostvars[host]['radosgw_address'] }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {% endif %}
+{%- else -%}
+    {% set interface = ["ansible_",radosgw_interface]|join %}
+    {% if ip_version == 'ipv6' -%}
+      rgw frontends = civetweb port=[{{ hostvars[host][interface][ip_version][0]['address'] }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {%- elif ip_version == 'ipv4' -%}
+      rgw frontends = civetweb port={{ hostvars[host][interface][ip_version]['address'] }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {% endif %}
+{%- endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endif %}
+{% endif %}
+
 {% if groups[restapi_group_name] is defined %}
 {% if restapi_group_name in group_names %}
 [client.restapi]

roles/ceph-nfs/defaults/main.yml

@@ -27,7 +27,7 @@ ceph_nfs_export_id: 20134
 ceph_nfs_pseudo_path: "/cephfile"
 ceph_nfs_protocols: "3,4"
 ceph_nfs_access_type: "RW"
-ceph_nfs_log_file: "/var/log/ganesha.log"
+ceph_nfs_log_file: "/var/log/ganesha/ganesha.log"
 
 ####################
 # FSAL Ceph Config #
@@ -49,6 +49,7 @@ ceph_nfs_rgw_user: "cephnfs"
 # they must be configered.
 #ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY"
 #ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C"
+rgw_client_name: client.rgw.{{ ansible_hostname }}
 
 ###################
 # CONFIG OVERRIDE #
@@ -66,6 +67,7 @@ ceph_nfs_rgw_user: "cephnfs"
 #
 #ganesha_ceph_export_overrides:
 #ganesha_rgw_export_overrides:
+#ganesha_rgw_section_overrides:
 #ganesha_log_overrides:
 #ganesha_conf_overrides: |
 #    CACHEINODE {

roles/ceph-nfs/tasks/pre_requisite.yml

@@ -1,4 +1,51 @@
 ---
+- name: create rados gateway directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "ceph"
+    group: "ceph"
+    mode: "0755"
+  with_items:
+    - /var/lib/ceph/bootstrap-rgw
+    - /var/lib/ceph/radosgw
+    - /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}
+    - "{{ rbd_client_admin_socket_path }}"
+  when:
+    - nfs_obj_gw
+
+- name: copy rados gateway bootstrap key
+  copy:
+    src: "{{ fetch_directory }}/{{ fsid }}{{ item.name }}"
+    dest: "{{ item.name }}"
+    owner: "ceph"
+    group: "ceph"
+    mode: "0600"
+  with_items:
+    - { name: "/var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring", copy_key: true }
+    - { name: "/etc/ceph/{{ cluster }}.client.admin.keyring", copy_key: "{{ copy_admin_key }}" }
+  when:
+    - nfs_obj_gw
+    - cephx
+    - item.copy_key|bool
+
+- name: create rados gateway keyring
+  command: ceph --cluster {{ cluster }} --name client.bootstrap-rgw --keyring /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring auth get-or-create client.rgw.{{ ansible_hostname }} osd 'allow rwx' mon 'allow rw' -o /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring
+  args:
+    creates: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring
+  changed_when: false
+  when:
+    - nfs_obj_gw
+    - cephx
+
+- name: set rados gateway key permissions
+  file:
+    path: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring
+    owner: "ceph"
+    group: "ceph"
+    mode: "0600"
+  when: cephx
+
 - name: create rgw nfs user
   command: radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name="RGW NFS User"
   register: rgwuser

roles/ceph-nfs/templates/ganesha.conf.j2

@@ -52,6 +52,13 @@ EXPORT
         {{ ganesha_rgw_export_overrides | default(None) }}
 
 }
+
+RGW {
+        ceph_conf = "/etc/ceph/{{ cluster }}.conf";
+        cluster = "{{ cluster }}";
+        name = "{{ rgw_client_name }}";
+        {{ ganesha_rgw_section_overrides | default(None) }}
+}
 {% endif %}
 
 LOG {

tests/functional/tests/nfs/test_nfs_ganesha.py

@@ -21,3 +21,12 @@ class TestNFSs(object):
     @pytest.mark.no_docker
     def test_nfs_config_override(self, node, host):
         assert host.file("/etc/ganesha/ganesha.conf").contains("Entries_HWMark")
+
+    @pytest.mark.no_docker
+    def test_nfs_rgw_fsal_export(self, node, host):
+        if(host.mount_point("/mnt").exists):
+            cmd = host.run("sudo umount /mnt")
+            assert cmd.rc == 0
+        cmd = host.run("sudo mount.nfs localhost:/ceph /mnt/")
+        assert cmd.rc == 0
+        assert host.mount_point("/mnt").exists