From c7d9838ad4bf5c8e39ce513626613a44b775e600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 19 Sep 2017 13:40:42 +0200 Subject: [PATCH 1/2] tests: add nfs container test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sébastien Han --- tests/functional/centos/7/docker/hosts | 3 +++ tests/functional/centos/7/docker/vagrant_variables.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/functional/centos/7/docker/hosts b/tests/functional/centos/7/docker/hosts index d63336efc..7e0ab98a1 100644 --- a/tests/functional/centos/7/docker/hosts +++ b/tests/functional/centos/7/docker/hosts @@ -15,6 +15,9 @@ rgw0 [mgrs] mgr0 +[nfss] +nfs0 + [clients] client0 ceph_origin="repository" ceph_repository="community" diff --git a/tests/functional/centos/7/docker/vagrant_variables.yml b/tests/functional/centos/7/docker/vagrant_variables.yml index 56e9cfe4c..5a49b01e4 100644 --- a/tests/functional/centos/7/docker/vagrant_variables.yml +++ b/tests/functional/centos/7/docker/vagrant_variables.yml @@ -8,7 +8,7 @@ mon_vms: 3 osd_vms: 1 mds_vms: 1 rgw_vms: 1 -nfs_vms: 0 +nfs_vms: 1 rbd_mirror_vms: 1 client_vms: 1 iscsi_gw_vms: 0 From aa5c36f19c93c86c930d5b7135c783a651b96e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 19 Sep 2017 16:19:14 +0200 Subject: [PATCH 2/2] nfs: several fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - move the file fetch/push to the existing task - rename the include - generate the ganesha template from ansible - re-arrange role structure - re-use tasks for non-container and container - configure keys for non-container and container - fix rgw container key collection; Signed-off-by: Sébastien Han --- group_vars/nfss.yml.sample | 2 +- roles/ceph-nfs/defaults/main.yml | 2 +- roles/ceph-nfs/tasks/create_rgw_nfs_user.yml | 29 ++++++++++ roles/ceph-nfs/tasks/docker/copy_configs.yml | 8 ++- .../tasks/docker/copy_ganesha_configs.yml | 37 ------------ roles/ceph-nfs/tasks/docker/main.yml | 10 +--- .../tasks/docker/start_docker_nfs.yml | 21 ++----- roles/ceph-nfs/tasks/main.yml | 14 ++++- roles/ceph-nfs/tasks/pre_requisite.yml | 57 +------------------ roles/ceph-nfs/tasks/start_nfs.yml | 20 +++++++ .../tasks/docker/copy_rgw_configs.yml | 39 ------------- roles/ceph-rgw/tasks/docker/main.yml | 3 - 12 files changed, 77 insertions(+), 165 deletions(-) create mode 100644 roles/ceph-nfs/tasks/create_rgw_nfs_user.yml delete mode 100644 roles/ceph-nfs/tasks/docker/copy_ganesha_configs.yml create mode 100644 roles/ceph-nfs/tasks/start_nfs.yml delete mode 100644 roles/ceph-rgw/tasks/docker/copy_rgw_configs.yml diff --git a/group_vars/nfss.yml.sample b/group_vars/nfss.yml.sample index 03b877253..eb0aa08a6 100644 --- a/group_vars/nfss.yml.sample +++ b/group_vars/nfss.yml.sample @@ -91,7 +91,7 @@ dummy: # DOCKER # ########## -#ceph_docker_image: "ceph/ganesha" +#ceph_docker_image: "ceph/daemon" #ceph_docker_image_tag: latest #ceph_nfs_docker_extra_env: #ceph_config_keys: [] # DON'T TOUCH ME diff --git a/roles/ceph-nfs/defaults/main.yml b/roles/ceph-nfs/defaults/main.yml index 320879ed3..6025d7e31 100644 --- a/roles/ceph-nfs/defaults/main.yml +++ b/roles/ceph-nfs/defaults/main.yml @@ -83,7 +83,7 @@ rgw_client_name: client.rgw.{{ ansible_hostname }} # DOCKER # ########## -ceph_docker_image: "ceph/ganesha" +ceph_docker_image: "ceph/daemon" ceph_docker_image_tag: latest ceph_nfs_docker_extra_env: ceph_config_keys: [] # DON'T TOUCH ME diff --git a/roles/ceph-nfs/tasks/create_rgw_nfs_user.yml b/roles/ceph-nfs/tasks/create_rgw_nfs_user.yml new file mode 100644 index 000000000..3c24fecf6 --- /dev/null +++ b/roles/ceph-nfs/tasks/create_rgw_nfs_user.yml @@ -0,0 +1,29 @@ +--- +- name: set_fact docker_exec_cmd_nfs + set_fact: + docker_exec_cmd_nfs: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}" + when: + - containerized_deployment + +- name: create rgw nfs user + command: "{{ docker_exec_cmd_nfs }} radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name='RGW NFS User'" + register: rgwuser + delegate_to: "{{ groups[mon_group_name][0] }}" + when: + - nfs_obj_gw + +- name: set_fact ceph_nfs_rgw_access_key + set_fact: + ceph_nfs_rgw_access_key: "{{ (rgwuser.stdout | from_json)['keys'][0]['access_key'] }}" + delegate_to: "{{ groups[mon_group_name][0] }}" + when: + - nfs_obj_gw + - ceph_nfs_rgw_access_key is not defined + +- name: set_fact ceph_nfs_rgw_secret_key + set_fact: + ceph_nfs_rgw_secret_key: "{{(rgwuser.stdout | from_json)['keys'][0]['secret_key']}}" + delegate_to: "{{ groups[mon_group_name][0] }}" + when: + - nfs_obj_gw + - ceph_nfs_rgw_secret_key is not defined diff --git a/roles/ceph-nfs/tasks/docker/copy_configs.yml b/roles/ceph-nfs/tasks/docker/copy_configs.yml index 9686111b1..9985d052b 100644 --- a/roles/ceph-nfs/tasks/docker/copy_configs.yml +++ b/roles/ceph-nfs/tasks/docker/copy_configs.yml @@ -3,7 +3,7 @@ set_fact: ceph_config_keys: - /etc/ceph/{{ cluster }}.client.admin.keyring - - /var/lib/ceph/radosgw/keyring + - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring - name: stat for config and keys local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }} @@ -25,7 +25,8 @@ with_together: - "{{ ceph_config_keys }}" - "{{ statconfig.results }}" - when: item.1.stat.exists == true + when: + - item.1.stat.exists == true - name: set selinux permissions shell: | @@ -34,4 +35,5 @@ - "{{ ceph_conf_key_directory }}" - /var/lib/ceph changed_when: false - when: sestatus.stdout != 'Disabled' + when: + - sestatus.stdout != 'Disabled' diff --git a/roles/ceph-nfs/tasks/docker/copy_ganesha_configs.yml b/roles/ceph-nfs/tasks/docker/copy_ganesha_configs.yml deleted file mode 100644 index 50de38e55..000000000 --- a/roles/ceph-nfs/tasks/docker/copy_ganesha_configs.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: set config paths - set_fact: - nfs_config_keys: - - /etc/ganesha/ganesha.conf - -- name: stat for config and keys - local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }} - with_items: "{{ nfs_config_keys }}" - changed_when: false - become: false - failed_when: false - always_run: true - register: statconfig - -- name: try to fetch config and keys - copy: - src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" - dest: "{{ item.0 }}" - owner: root - group: root - mode: 0644 - changed_when: false - with_together: - - "{{ nfs_config_keys }}" - - "{{ statconfig.results }}" - when: item.1.stat.exists == true - -- name: push ganesha files to the ansible server - fetch: - src: "{{ item.0 }}" - dest: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" - flat: yes - with_together: - - "{{ nfs_config_keys }}" - - "{{ statconfig.results }}" - when: item.1.stat.exists == false diff --git a/roles/ceph-nfs/tasks/docker/main.yml b/roles/ceph-nfs/tasks/docker/main.yml index dbf56263b..f05ce4ae9 100644 --- a/roles/ceph-nfs/tasks/docker/main.yml +++ b/roles/ceph-nfs/tasks/docker/main.yml @@ -1,11 +1,3 @@ --- -- name: include copy_configs.yml - include: copy_configs.yml - -- name: copy_ganesha_configs.yml - include: copy_ganesha_configs.yml - when: not containerized_deployment_with_kv - -- name: start_docker_nfs.yml +- name: include start_docker_nfs.yml include: start_docker_nfs.yml - diff --git a/roles/ceph-nfs/tasks/docker/start_docker_nfs.yml b/roles/ceph-nfs/tasks/docker/start_docker_nfs.yml index cdf9bb0c5..45bc18eb2 100644 --- a/roles/ceph-nfs/tasks/docker/start_docker_nfs.yml +++ b/roles/ceph-nfs/tasks/docker/start_docker_nfs.yml @@ -8,22 +8,11 @@ group: "root" mode: "0644" -- name: enable systemd unit file for nfs instance - shell: systemctl enable ceph-nfs@{{ ansible_hostname }}.service - failed_when: false - changed_when: false - when: ceph_nfs_enable_service - -- name: reload systemd unit files - shell: systemctl daemon-reload - changed_when: false - failed_when: false - when: ceph_nfs_enable_service - - name: systemd start nfs container - service: - name: ceph-nfs@{{ ansible_hostname }} + systemd: + name: "ceph-nfs@{{ ansible_hostname }}.service" state: started enabled: yes - changed_when: false - when: ceph_nfs_enable_service + daemon_reload: yes + when: + - ceph_nfs_enable_service diff --git a/roles/ceph-nfs/tasks/main.yml b/roles/ceph-nfs/tasks/main.yml index f2b42763c..fe70f3c40 100644 --- a/roles/ceph-nfs/tasks/main.yml +++ b/roles/ceph-nfs/tasks/main.yml @@ -1,8 +1,18 @@ --- - name: include pre_requisite.yml include: pre_requisite.yml - when: not containerized_deployment + when: + - not containerized_deployment + +- name: include create_rgw_nfs_user.yml + include: create_rgw_nfs_user.yml + +- name: include start_nfs.yml + include: start_nfs.yml + when: + - not containerized_deployment - name: include docker/main.yml include: docker/main.yml - when: containerized_deployment + when: + - containerized_deployment diff --git a/roles/ceph-nfs/tasks/pre_requisite.yml b/roles/ceph-nfs/tasks/pre_requisite.yml index 9d4bb2f0d..afe1c7094 100644 --- a/roles/ceph-nfs/tasks/pre_requisite.yml +++ b/roles/ceph-nfs/tasks/pre_requisite.yml @@ -1,5 +1,5 @@ --- -- name: create rados gateway directories +- name: create rados gateway and ganesha directories file: path: "{{ item }}" state: directory @@ -11,6 +11,8 @@ - /var/lib/ceph/radosgw - /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }} - "{{ rbd_client_admin_socket_path }}" + - /var/lib/nfs/ganesha + - /var/run/ganesha when: - nfs_obj_gw @@ -47,62 +49,9 @@ when: - cephx -- name: create rgw nfs user - command: radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name="RGW NFS User" - register: rgwuser - delegate_to: "{{ groups[mon_group_name][0] }}" - when: - - nfs_obj_gw - -- name: set_fact ceph_nfs_rgw_access_key - set_fact: - ceph_nfs_rgw_access_key: "{{ (rgwuser.stdout | from_json)['keys'][0]['access_key'] }}" - delegate_to: "{{ groups[mon_group_name][0] }}" - when: - - nfs_obj_gw - - ceph_nfs_rgw_access_key is not defined - -- name: set_fact ceph_nfs_rgw_secret_key - set_fact: - ceph_nfs_rgw_secret_key: "{{(rgwuser.stdout | from_json)['keys'][0]['secret_key']}}" - delegate_to: "{{ groups[mon_group_name][0] }}" - when: - - nfs_obj_gw - - ceph_nfs_rgw_secret_key is not defined - -- name: create nfs gateway directories - file: - path: "{{ item }}" - state: directory - owner: "ceph" - group: "ceph" - mode: "0755" - with_items: - - /var/lib/nfs/ganesha - - /var/run/ganesha - -- name: generate ganesha configuration file - action: config_template - args: - src: "{{ lookup('env', 'ANSIBLE_ROLES_PATH') | default (playbook_dir + '/roles', true) }}/ceph-nfs/templates/ganesha.conf.j2" - dest: /etc/ganesha/ganesha.conf - owner: "root" - group: "root" - mode: "0644" - config_type: ini - notify: - - restart ceph nfss - - name: change ownership on /var/log/ganesha file: path: '/var/log/ganesha' owner: 'root' group: 'root' mode: '0755' - -- name: start nfs gateway service - service: - name: nfs-ganesha - state: started - enabled: yes - when: ceph_nfs_enable_service diff --git a/roles/ceph-nfs/tasks/start_nfs.yml b/roles/ceph-nfs/tasks/start_nfs.yml new file mode 100644 index 000000000..02513149d --- /dev/null +++ b/roles/ceph-nfs/tasks/start_nfs.yml @@ -0,0 +1,20 @@ +--- +- name: generate ganesha configuration file + action: config_template + args: + src: "{{ lookup('env', 'ANSIBLE_ROLES_PATH') | default (playbook_dir + '/roles', true) }}/ceph-nfs/templates/ganesha.conf.j2" + dest: /etc/ganesha/ganesha.conf + owner: "root" + group: "root" + mode: "0644" + config_type: ini + notify: + - restart ceph nfss + +- name: start nfs gateway service + service: + name: nfs-ganesha + state: started + enabled: yes + when: + - ceph_nfs_enable_service diff --git a/roles/ceph-rgw/tasks/docker/copy_rgw_configs.yml b/roles/ceph-rgw/tasks/docker/copy_rgw_configs.yml deleted file mode 100644 index cda9f789c..000000000 --- a/roles/ceph-rgw/tasks/docker/copy_rgw_configs.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: set_fact rgw_config_keys - set_fact: - rgw_config_keys: - - "/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring" - when: - - nfs_obj_gw - -- name: wait for rgw keyring - wait_for: - path: "/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring" - when: - - nfs_obj_gw - - inventory_hostname == groups[rgw_group_name][0] - -- name: stat for config and keys - local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }} - with_items: "{{ rgw_config_keys }}" - changed_when: false - become: false - failed_when: false - always_run: true - register: statconfig - when: - - nfs_obj_gw - - inventory_hostname == groups[rgw_group_name][0] - -- name: push ceph files to the ansible server - fetch: - src: "{{ item.0 }}" - dest: "{{ fetch_directory }}/{{ fsid }}/var/lib/ceph/radosgw/keyring" - flat: yes - with_together: - - "{{ rgw_config_keys }}" - - "{{ statconfig.results }}" - when: - - nfs_obj_gw - - item.1.stat.exists == false - - inventory_hostname == groups[rgw_group_name][0] diff --git a/roles/ceph-rgw/tasks/docker/main.yml b/roles/ceph-rgw/tasks/docker/main.yml index 80bcb0109..bad0b065e 100644 --- a/roles/ceph-rgw/tasks/docker/main.yml +++ b/roles/ceph-rgw/tasks/docker/main.yml @@ -4,6 +4,3 @@ - name: include start_docker_rgw.yml include: start_docker_rgw.yml - -- name: include copy_rgw_configs.yml - include: copy_rgw_configs.yml