From 2fd9bbbe09d875805b000b5cd0da689c2f8497dd Mon Sep 17 00:00:00 2001 From: Daniel Gryniewicz Date: Thu, 21 Jul 2016 09:17:19 -0400 Subject: [PATCH] NFS for FSAL_RGW Add support for FSAL_RGW to the NFS gateway. Both standard and containerized versions are supported. --- group_vars/nfss.sample | 22 ++++++++++ .../tasks/installs/install_on_redhat.yml | 30 +++++++++++-- roles/ceph-common/templates/ganesha.conf.j2 | 43 +++++++++++++++++-- roles/ceph-nfs/defaults/main.yml | 22 ++++++++++ roles/ceph-nfs/tasks/docker/copy_configs.yml | 30 ++++++++++--- .../ceph-nfs/tasks/docker/create_configs.yml | 15 +++++++ .../tasks/docker/dirs_permissions.yml | 2 + roles/ceph-nfs/tasks/docker/fetch_configs.yml | 19 +++++--- roles/ceph-nfs/tasks/docker/main.yml | 11 +++-- roles/ceph-nfs/tasks/pre_requisite.yml | 30 ++++++++++++- roles/ceph-nfs/templates/ceph-nfs.service.j2 | 5 +-- .../tasks/docker/start_docker_osd.yml | 4 +- roles/ceph-rgw/tasks/docker/copy_configs.yml | 36 ++++++++++++++++ roles/ceph-rgw/tasks/docker/main.yml | 2 + 14 files changed, 242 insertions(+), 29 deletions(-) create mode 100644 roles/ceph-rgw/tasks/docker/copy_configs.yml diff --git a/group_vars/nfss.sample b/group_vars/nfss.sample index 8e378d247..027949468 100644 --- a/group_vars/nfss.sample +++ b/group_vars/nfss.sample @@ -35,6 +35,28 @@ dummy: #ceph_nfs_pseudo_path: "/ceph" #ceph_nfs_protocols: "3,4" #ceph_nfs_access_type: "RW" +#ceph_nfs_log_file: "/var/log/ganesha.log" + +#################### +# FSAL Ceph Config # +#################### +#ceph_nfs_ceph_export_id: 20134 +#ceph_nfs_ceph_pseudo_path: "/ceph" +#ceph_nfs_ceph_protocols: "3,4" +#ceph_nfs_ceph_access_type: "RW" + +################### +# FSAL RGW Config # +################### +#ceph_nfs_rgw_export_id: 20134 +#ceph_nfs_rgw_pseudo_path: "/ceph" +#ceph_nfs_rgw_protocols: "3,4" +#ceph_nfs_rgw_access_type: "RW" +#ceph_nfs_rgw_user: "cephnfs" +# Note: keys are optional and can be generated, but not on containerized, where +# they must be configered. +#ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY" +#ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C" ################### diff --git a/roles/ceph-common/tasks/installs/install_on_redhat.yml b/roles/ceph-common/tasks/installs/install_on_redhat.yml index 56b3c9121..528358832 100644 --- a/roles/ceph-common/tasks/installs/install_on_redhat.yml +++ b/roles/ceph-common/tasks/installs/install_on_redhat.yml @@ -227,18 +227,40 @@ - rgw_group_name in group_names - ansible_pkg_mgr == "dnf" -- name: install NFS gateway +- name: install nfs ceph gateway yum: name: nfs-ganesha-ceph - state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" when: - nfs_group_name in group_names - ansible_pkg_mgr == "yum" + - fsal_ceph -- name: install NFS gateway +- name: install nfs ceph gateway dnf: name: nfs-ganesha-ceph - state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" when: - nfs_group_name in group_names - ansible_pkg_mgr == "dnf" + - fsal_ceph + +- name: install nfs rgw gateway + yum: + name: "{{ item }}" + with_items: + - nfs-ganesha-rgw + - ceph-radosgw + when: + - nfs_group_name in group_names + - ansible_pkg_mgr == "yum" + - fsal_rgw + +- name: install nfs rgw gateway + dnf: + name: "{{ item }}" + with_items: + - nfs-ganesha-rgw + - ceph-radosgw + when: + - nfs_group_name in group_names + - ansible_pkg_mgr == "dnf" + - fsal_rgw diff --git a/roles/ceph-common/templates/ganesha.conf.j2 b/roles/ceph-common/templates/ganesha.conf.j2 index ab64ea6a3..38bb67b0b 100644 --- a/roles/ceph-common/templates/ganesha.conf.j2 +++ b/roles/ceph-common/templates/ganesha.conf.j2 @@ -1,17 +1,18 @@ #jinja2: trim_blocks: "true", lstrip_blocks: "true" # {{ ansible_managed }} +{% if fsal_ceph %} EXPORT { - Export_ID={{ ceph_nfs_export_id }}; + Export_ID={{ ceph_nfs_ceph_export_id }}; Path = "/"; - Pseudo = {{ ceph_nfs_pseudo_path }}; + Pseudo = {{ ceph_nfs_ceph_pseudo_path }}; - Access_Type = {{ ceph_nfs_access_type }}; + Access_Type = {{ ceph_nfs_ceph_access_type }}; - NFS_Protocols = {{ ceph_nfs_protocols }}; + NFS_Protocols = {{ ceph_nfs_ceph_protocols }}; Transport_Protocols = TCP; @@ -21,3 +22,37 @@ EXPORT Name = CEPH; } } +{% endif %} +{% if fsal_rgw %} +EXPORT +{ + Export_ID={{ ceph_nfs_rgw_export_id }}; + + Path = "/"; + + Pseudo = {{ ceph_nfs_rgw_pseudo_path }}; + + Access_Type = {{ ceph_nfs_rgw_access_type }}; + + NFS_Protocols = {{ ceph_nfs_rgw_protocols }}; + + Transport_Protocols = TCP; + + Sectype = sys,krb5,krb5i,krb5p; + + FSAL { + Name = RGW; + User_Id = "{{ ceph_nfs_rgw_user }}"; + Access_Key_Id ="{{ ceph_nfs_rgw_access_key }}"; + Secret_Access_Key = "{{ ceph_nfs_rgw_secret_key }}"; + } +} +{% endif %} + +LOG { + Facility { + name = FILE; + destination = "{{ ceph_nfs_log_file }}"; + enable = active; + } +} diff --git a/roles/ceph-nfs/defaults/main.yml b/roles/ceph-nfs/defaults/main.yml index 64f643e5e..2848361ef 100644 --- a/roles/ceph-nfs/defaults/main.yml +++ b/roles/ceph-nfs/defaults/main.yml @@ -27,6 +27,28 @@ ceph_nfs_export_id: 20134 ceph_nfs_pseudo_path: "/ceph" ceph_nfs_protocols: "3,4" ceph_nfs_access_type: "RW" +ceph_nfs_log_file: "/var/log/ganesha.log" + +#################### +# FSAL Ceph Config # +#################### +ceph_nfs_ceph_export_id: 20134 +ceph_nfs_ceph_pseudo_path: "/ceph" +ceph_nfs_ceph_protocols: "3,4" +ceph_nfs_ceph_access_type: "RW" + +################### +# FSAL RGW Config # +################### +ceph_nfs_rgw_export_id: 20134 +ceph_nfs_rgw_pseudo_path: "/ceph" +ceph_nfs_rgw_protocols: "3,4" +ceph_nfs_rgw_access_type: "RW" +ceph_nfs_rgw_user: "cephnfs" +# Note: keys are optional and can be generated, but not on containerized, where +# they must be configered. +#ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY" +#ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C" ################### diff --git a/roles/ceph-nfs/tasks/docker/copy_configs.yml b/roles/ceph-nfs/tasks/docker/copy_configs.yml index f2ba50e81..8f1f3835a 100644 --- a/roles/ceph-nfs/tasks/docker/copy_configs.yml +++ b/roles/ceph-nfs/tasks/docker/copy_configs.yml @@ -1,10 +1,28 @@ --- -- name: push ceph files to the ansible server - fetch: - src: "{{ item.0 }}" - dest: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" - flat: yes +- name: set config and keys paths + set_fact: + ceph_config_keys: + - /etc/ceph/ceph.conf + - /etc/ceph/ceph.client.admin.keyring + - /var/lib/ceph/radosgw/keyring + +- name: stat for config and keys + local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }} + with_items: ceph_config_keys + changed_when: false + become: false + failed_when: false + register: statconfig + +- name: try to fetch config and keys + copy: + src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + dest: "{{ item.0 }}" + owner: "64045" + group: "64045" + mode: 0644 + changed_when: false with_together: - ceph_config_keys - statconfig.results - when: item.1.stat.exists == false + when: item.1.stat.exists == true diff --git a/roles/ceph-nfs/tasks/docker/create_configs.yml b/roles/ceph-nfs/tasks/docker/create_configs.yml index 3a9b51ccc..560ff20fc 100644 --- a/roles/ceph-nfs/tasks/docker/create_configs.yml +++ b/roles/ceph-nfs/tasks/docker/create_configs.yml @@ -7,6 +7,21 @@ group: root mode: 0644 +- name: create the nfs rgw user + docker: + image: "{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}:{{ ceph_rgw_docker_image_tag }}" + name: ceph-{{ ansible_hostname }}-rgw-user + expose: "{{ ceph_rgw_civetweb_port }}" + ports: "{{ ceph_rgw_civetweb_port }}:{{ ceph_rgw_civetweb_port }}" + state: running + env: "CEPH_DAEMON=RGW_USER,RGW_USERNAME={{ ceph_nfs_rgw_user }},RGW_USER_ACCESS_KEY={{ ceph_nfs_rgw_access_key }},RGW_USER_SECRET_KEY={{ ceph_nfs_rgw_secret_key }}" + volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" + when: fsal_rgw + +- name: get user create output + command: docker logs ceph-{{ ansible_hostname }}-rgw-user + register: rgwuser + - name: generate ganesha configuration file action: config_template args: diff --git a/roles/ceph-nfs/tasks/docker/dirs_permissions.yml b/roles/ceph-nfs/tasks/docker/dirs_permissions.yml index 87ca765d7..b83f36b8c 100644 --- a/roles/ceph-nfs/tasks/docker/dirs_permissions.yml +++ b/roles/ceph-nfs/tasks/docker/dirs_permissions.yml @@ -22,6 +22,7 @@ with_items: - /etc/ceph/ - /var/lib/ceph/ + - /var/lib/ceph/radosgw when: not after_hammer - name: create bootstrap directories (after hammer) @@ -34,6 +35,7 @@ with_items: - /etc/ceph/ - /var/lib/ceph/ + - /var/lib/ceph/radosgw when: after_hammer - name: create ganesha directories diff --git a/roles/ceph-nfs/tasks/docker/fetch_configs.yml b/roles/ceph-nfs/tasks/docker/fetch_configs.yml index bd7f746ae..07e6a40d2 100644 --- a/roles/ceph-nfs/tasks/docker/fetch_configs.yml +++ b/roles/ceph-nfs/tasks/docker/fetch_configs.yml @@ -1,13 +1,12 @@ --- -- name: set config and keys paths +- name: set config paths set_fact: - ceph_config_keys: - - /etc/ceph/ceph.conf + nfs_config_keys: - /etc/ganesha/ganesha.conf - name: stat for config and keys local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }} - with_items: ceph_config_keys + with_items: nfs_config_keys changed_when: false become: false failed_when: false @@ -22,6 +21,16 @@ mode: 0644 changed_when: false with_together: - - ceph_config_keys + - nfs_config_keys - statconfig.results when: item.1.stat.exists == true + +- name: push ganesha files to the ansible server + fetch: + src: "{{ item.0 }}" + dest: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + flat: yes + with_together: + - nfs_config_keys + - statconfig.results + when: item.1.stat.exists == false diff --git a/roles/ceph-nfs/tasks/docker/main.yml b/roles/ceph-nfs/tasks/docker/main.yml index 1e4d030f9..2507d11de 100644 --- a/roles/ceph-nfs/tasks/docker/main.yml +++ b/roles/ceph-nfs/tasks/docker/main.yml @@ -45,12 +45,19 @@ - include: dirs_permissions.yml -# let the first ganesha create configs and keyrings +# Copy Ceph configs to host +- include: copy_configs.yml + +- include: selinux.yml + when: ansible_os_family == 'RedHat' + +# let the first ganesha create configs and users - include: create_configs.yml when: inventory_hostname == groups.nfss[0] and mon_containerized_default_ceph_conf_with_kv +# Copy Ganesha configs to host - include: fetch_configs.yml when: not mon_containerized_deployment_with_kv @@ -59,5 +66,3 @@ - include: start_docker_nfs.yml -- include: copy_configs.yml - when: not mon_containerized_deployment_with_kv diff --git a/roles/ceph-nfs/tasks/pre_requisite.yml b/roles/ceph-nfs/tasks/pre_requisite.yml index 6119cd40b..695f93cab 100644 --- a/roles/ceph-nfs/tasks/pre_requisite.yml +++ b/roles/ceph-nfs/tasks/pre_requisite.yml @@ -1,5 +1,5 @@ --- -- name: create NFS gateway directories +- name: create nfs gateway directories file: path: "{{ item }}" state: directory @@ -10,7 +10,33 @@ - /var/lib/nfs/ganesha - /var/run/ganesha -- name: start NFS gateway service +- name: create rgw nfs user + command: radosgw-admin user create --uid={{ ceph_nfs_rgw_user }} --display-name="RGW NFS User" + register: rgwuser + when: fsal_rgw + +- name: set access key + set_fact: + ceph_nfs_rgw_access_key: "{{ (rgwuser.stdout | from_json)['keys'][0]['access_key'] }}" + when: fsal_rgw + +- name: set secret key + set_fact: + ceph_nfs_rgw_secret_key: "{{(rgwuser.stdout | from_json)['keys'][0]['secret_key']}}" + when: fsal_rgw + +- name: generate ganesha configuration file + action: config_template + args: + src: "{{ playbook_dir }}/roles/ceph-common/templates/ganesha.conf.j2" + dest: /etc/ganesha/ganesha.conf + owner: "root" + group: "root" + mode: "0644" + config_overrides: "{{ ganesha_conf_overrides }}" + config_type: ini + +- name: start nfs gateway service service: name: nfs-ganesha state: started diff --git a/roles/ceph-nfs/templates/ceph-nfs.service.j2 b/roles/ceph-nfs/templates/ceph-nfs.service.j2 index bd8b41b0a..023bcfa77 100644 --- a/roles/ceph-nfs/templates/ceph-nfs.service.j2 +++ b/roles/ceph-nfs/templates/ceph-nfs.service.j2 @@ -7,7 +7,7 @@ After=docker.service EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/docker rm %i ExecStartPre=/usr/bin/mkdir -p /etc/ceph /etc/ganesha /var/lib/nfs/ganesha -ExecStart=/usr/bin/docker run --rm --name %i --net=host \ +ExecStart=/usr/bin/docker run --rm --net=host \ {% if not mon_containerized_deployment_with_kv -%} -v /etc/ceph:/etc/ceph \ -v /etc/ganesha:/etc/ganesha \ @@ -18,8 +18,7 @@ ExecStart=/usr/bin/docker run --rm --name %i --net=host \ -v /etc/localtime:/etc/localtime:ro \ --privileged \ -e CEPH_DAEMON=NFS \ - -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \ - --name={{ ansible_hostname }} \ + --name=nfs-{{ ansible_hostname }} \ {{ ceph_nfs_docker_username }}/{{ ceph_nfs_docker_imagename }}:{{ ceph_nfs_docker_image_tag }} ExecStopPost=-/usr/bin/docker stop %i Restart=always diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index f466f02b8..9f34a9811 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -31,8 +31,8 @@ -v /etc/localtime:/etc/localtime:ro \ -e "OSD_DEVICE={{ item.0 }}" \ -e "{{ ceph_osd_docker_prepare_env }}" \ - "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}" \ - OSD_CEPH_DISK_PREPARE + -e CEPH_DAEMON=osd_ceph_disk_prepare \ + "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}" with_together: - ceph_osd_docker_devices - osd_prepared.results diff --git a/roles/ceph-rgw/tasks/docker/copy_configs.yml b/roles/ceph-rgw/tasks/docker/copy_configs.yml new file mode 100644 index 000000000..3345f1803 --- /dev/null +++ b/roles/ceph-rgw/tasks/docker/copy_configs.yml @@ -0,0 +1,36 @@ +--- +- name: set config and keys paths + set_fact: + rgw_config_keys: + - "/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring" + when: fsal_rgw + +- name: wait for rgw keyring + wait_for: path="/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring" + when: + - fsal_rgw + - inventory_hostname == groups.rgws[0] + +- name: stat for config and keys + local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }} + with_items: rgw_config_keys + changed_when: false + become: false + failed_when: false + register: statconfig + when: + - fsal_rgw + - inventory_hostname == groups.rgws[0] + +- name: push ceph files to the ansible server + fetch: + src: "{{ item.0 }}" + dest: "{{ fetch_directory }}/docker_mon_files/var/lib/ceph/radosgw/keyring" + flat: yes + with_together: + - rgw_config_keys + - statconfig.results + when: + - item.1.stat.exists == false + - fsal_rgw + - inventory_hostname == groups.rgws[0] diff --git a/roles/ceph-rgw/tasks/docker/main.yml b/roles/ceph-rgw/tasks/docker/main.yml index 360d37b71..63579ed8a 100644 --- a/roles/ceph-rgw/tasks/docker/main.yml +++ b/roles/ceph-rgw/tasks/docker/main.yml @@ -49,3 +49,5 @@ when: ansible_os_family == 'RedHat' - include: start_docker_rgw.yml + +- include: copy_configs.yml