Merge pull request #1923 from ceph/nfs-container

[skip ci] tests: add nfs container test

group_vars/nfss.yml.sample

@@ -91,7 +91,7 @@ dummy:
 # DOCKER #
 ##########
 
-#ceph_docker_image: "ceph/ganesha"
+#ceph_docker_image: "ceph/daemon"
 #ceph_docker_image_tag: latest
 #ceph_nfs_docker_extra_env:
 #ceph_config_keys: [] # DON'T TOUCH ME

roles/ceph-nfs/defaults/main.yml

@@ -83,7 +83,7 @@ rgw_client_name: client.rgw.{{ ansible_hostname }}
 # DOCKER #
 ##########
 
-ceph_docker_image: "ceph/ganesha"
+ceph_docker_image: "ceph/daemon"
 ceph_docker_image_tag: latest
 ceph_nfs_docker_extra_env:
 ceph_config_keys: [] # DON'T TOUCH ME

roles/ceph-nfs/tasks/create_rgw_nfs_user.yml

@@ -0,0 +1,29 @@
+---
+- name: set_fact docker_exec_cmd_nfs
+  set_fact:
+    docker_exec_cmd_nfs: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
+  when:
+    - containerized_deployment
+
+- name: create rgw nfs user
+  command: "{{ docker_exec_cmd_nfs }} radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name='RGW NFS User'"
+  register: rgwuser
+  delegate_to: "{{ groups[mon_group_name][0] }}"
+  when:
+    - nfs_obj_gw
+
+- name: set_fact ceph_nfs_rgw_access_key
+  set_fact:
+    ceph_nfs_rgw_access_key: "{{ (rgwuser.stdout | from_json)['keys'][0]['access_key'] }}"
+  delegate_to: "{{ groups[mon_group_name][0] }}"
+  when:
+    - nfs_obj_gw
+    - ceph_nfs_rgw_access_key is not defined
+
+- name: set_fact ceph_nfs_rgw_secret_key
+  set_fact:
+    ceph_nfs_rgw_secret_key: "{{(rgwuser.stdout | from_json)['keys'][0]['secret_key']}}"
+  delegate_to: "{{ groups[mon_group_name][0] }}"
+  when:
+    - nfs_obj_gw
+    - ceph_nfs_rgw_secret_key is not defined

roles/ceph-nfs/tasks/docker/copy_configs.yml

@@ -3,7 +3,7 @@
   set_fact:
     ceph_config_keys:
       - /etc/ceph/{{ cluster }}.client.admin.keyring
-      - /var/lib/ceph/radosgw/keyring
+      - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring
 
 - name: stat for config and keys
   local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
@@ -25,7 +25,8 @@
   with_together:
     - "{{ ceph_config_keys }}"
     - "{{ statconfig.results }}"
-  when: item.1.stat.exists == true
+  when:
+    - item.1.stat.exists == true
 
 - name: set selinux permissions
   shell: |
@@ -34,4 +35,5 @@
     - "{{ ceph_conf_key_directory }}"
     - /var/lib/ceph
   changed_when: false
-  when: sestatus.stdout != 'Disabled'
+  when:
+    - sestatus.stdout != 'Disabled'

roles/ceph-nfs/tasks/docker/copy_ganesha_configs.yml

@@ -1,37 +0,0 @@
----
-- name: set config paths
-  set_fact:
-    nfs_config_keys:
-      - /etc/ganesha/ganesha.conf
-
-- name: stat for config and keys
-  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
-  with_items: "{{ nfs_config_keys }}"
-  changed_when: false
-  become: false
-  failed_when: false
-  always_run: true
-  register: statconfig
-
-- name: try to fetch config and keys
-  copy:
-    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
-    dest: "{{ item.0 }}"
-    owner: root
-    group: root
-    mode: 0644
-  changed_when: false
-  with_together:
-    - "{{ nfs_config_keys }}"
-    - "{{ statconfig.results }}"
-  when: item.1.stat.exists == true
-
-- name: push ganesha files to the ansible server
-  fetch:
-    src: "{{ item.0 }}"
-    dest: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
-    flat: yes
-  with_together:
-    - "{{ nfs_config_keys }}"
-    - "{{ statconfig.results }}"
-  when: item.1.stat.exists == false

roles/ceph-nfs/tasks/docker/main.yml

@@ -1,11 +1,3 @@
 ---
-- name: include copy_configs.yml
+- name: include start_docker_nfs.yml
-  include: copy_configs.yml
-
-- name: copy_ganesha_configs.yml
-  include: copy_ganesha_configs.yml
-  when: not containerized_deployment_with_kv
-
-- name: start_docker_nfs.yml
   include: start_docker_nfs.yml
-

roles/ceph-nfs/tasks/docker/start_docker_nfs.yml

@@ -8,22 +8,11 @@
     group: "root"
     mode: "0644"
 
-- name: enable systemd unit file for nfs instance
-  shell: systemctl enable ceph-nfs@{{ ansible_hostname }}.service
-  failed_when: false
-  changed_when: false
-  when: ceph_nfs_enable_service
-
-- name: reload systemd unit files
-  shell: systemctl daemon-reload
-  changed_when: false
-  failed_when: false
-  when: ceph_nfs_enable_service
-
 - name: systemd start nfs container
-  service:
+  systemd:
-    name: ceph-nfs@{{ ansible_hostname }}
+    name: "ceph-nfs@{{ ansible_hostname }}.service"
     state: started
     enabled: yes
-  changed_when: false
+    daemon_reload: yes
-  when: ceph_nfs_enable_service
+  when:
+    - ceph_nfs_enable_service

roles/ceph-nfs/tasks/main.yml

@@ -1,8 +1,18 @@
 ---
 - name: include pre_requisite.yml
   include: pre_requisite.yml
-  when: not containerized_deployment
+  when:
+    - not containerized_deployment
+
+- name: include create_rgw_nfs_user.yml
+  include: create_rgw_nfs_user.yml
+
+- name: include start_nfs.yml
+  include: start_nfs.yml
+  when:
+    - not containerized_deployment
 
 - name: include docker/main.yml
   include: docker/main.yml
-  when: containerized_deployment
+  when:
+    - containerized_deployment

roles/ceph-nfs/tasks/pre_requisite.yml

@@ -1,5 +1,5 @@
 ---
-- name: create rados gateway directories
+- name: create rados gateway and ganesha directories
   file:
     path: "{{ item }}"
     state: directory
@@ -11,6 +11,8 @@
     - /var/lib/ceph/radosgw
     - /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}
     - "{{ rbd_client_admin_socket_path }}"
+    - /var/lib/nfs/ganesha
+    - /var/run/ganesha
   when:
     - nfs_obj_gw
 
@@ -47,62 +49,9 @@
   when:
     - cephx
 
-- name: create rgw nfs user
-  command: radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name="RGW NFS User"
-  register: rgwuser
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  when:
-    - nfs_obj_gw
-
-- name: set_fact ceph_nfs_rgw_access_key
-  set_fact:
-    ceph_nfs_rgw_access_key: "{{ (rgwuser.stdout | from_json)['keys'][0]['access_key'] }}"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  when:
-    - nfs_obj_gw
-    - ceph_nfs_rgw_access_key is not defined
-
-- name: set_fact ceph_nfs_rgw_secret_key
-  set_fact:
-    ceph_nfs_rgw_secret_key: "{{(rgwuser.stdout | from_json)['keys'][0]['secret_key']}}"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  when:
-    - nfs_obj_gw
-    - ceph_nfs_rgw_secret_key is not defined
-
-- name: create nfs gateway directories
-  file:
-    path: "{{ item }}"
-    state: directory
-    owner: "ceph"
-    group: "ceph"
-    mode: "0755"
-  with_items:
-    - /var/lib/nfs/ganesha
-    - /var/run/ganesha
-
-- name: generate ganesha configuration file
-  action: config_template
-  args:
-    src: "{{ lookup('env', 'ANSIBLE_ROLES_PATH') | default (playbook_dir + '/roles', true) }}/ceph-nfs/templates/ganesha.conf.j2"
-    dest: /etc/ganesha/ganesha.conf
-    owner: "root"
-    group: "root"
-    mode: "0644"
-    config_type: ini
-  notify:
-    - restart ceph nfss
-
 - name: change ownership on /var/log/ganesha
   file:
     path: '/var/log/ganesha'
     owner: 'root'
     group: 'root'
     mode: '0755'
-
-- name: start nfs gateway service
-  service:
-    name: nfs-ganesha
-    state: started
-    enabled: yes
-  when: ceph_nfs_enable_service

roles/ceph-nfs/tasks/start_nfs.yml

@@ -0,0 +1,20 @@
+---
+- name: generate ganesha configuration file
+  action: config_template
+  args:
+    src: "{{ lookup('env', 'ANSIBLE_ROLES_PATH') | default (playbook_dir + '/roles', true) }}/ceph-nfs/templates/ganesha.conf.j2"
+    dest: /etc/ganesha/ganesha.conf
+    owner: "root"
+    group: "root"
+    mode: "0644"
+    config_type: ini
+  notify:
+    - restart ceph nfss
+
+- name: start nfs gateway service
+  service:
+    name: nfs-ganesha
+    state: started
+    enabled: yes
+  when:
+    - ceph_nfs_enable_service

roles/ceph-rgw/tasks/docker/copy_rgw_configs.yml

@@ -1,39 +0,0 @@
----
-- name: set_fact rgw_config_keys
-  set_fact:
-    rgw_config_keys:
-      - "/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring"
-  when:
-    - nfs_obj_gw
-
-- name: wait for rgw keyring
-  wait_for:
-    path: "/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring"
-  when:
-    - nfs_obj_gw
-    - inventory_hostname == groups[rgw_group_name][0]
-
-- name: stat for config and keys
-  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
-  with_items: "{{ rgw_config_keys }}"
-  changed_when: false
-  become: false
-  failed_when: false
-  always_run: true
-  register: statconfig
-  when:
-    - nfs_obj_gw
-    - inventory_hostname == groups[rgw_group_name][0]
-
-- name: push ceph files to the ansible server
-  fetch:
-    src: "{{ item.0 }}"
-    dest: "{{ fetch_directory }}/{{ fsid }}/var/lib/ceph/radosgw/keyring"
-    flat: yes
-  with_together:
-    - "{{ rgw_config_keys }}"
-    - "{{ statconfig.results }}"
-  when:
-    - nfs_obj_gw
-    - item.1.stat.exists == false
-    - inventory_hostname == groups[rgw_group_name][0]

roles/ceph-rgw/tasks/docker/main.yml

@@ -4,6 +4,3 @@
 
 - name: include start_docker_rgw.yml
   include: start_docker_rgw.yml
-
-- name: include copy_rgw_configs.yml
-  include: copy_rgw_configs.yml

tests/functional/centos/7/docker/hosts

@@ -15,6 +15,9 @@ rgw0
 [mgrs]
 mgr0
 
+[nfss]
+nfs0
+
 [clients]
 client0 ceph_origin="repository" ceph_repository="community"
 

tests/functional/centos/7/docker/vagrant_variables.yml

@@ -8,7 +8,7 @@ mon_vms: 3
 osd_vms: 1
 mds_vms: 1
 rgw_vms: 1
-nfs_vms: 0
+nfs_vms: 1
 rbd_mirror_vms: 1
 client_vms: 1
 iscsi_gw_vms: 0