Merge pull request #1780 from ceph/wip-rgw-nfs

Wip RGW NFS
pull/1875/head
Sébastien Han 2017-09-08 19:26:02 +02:00 committed by GitHub
commit d46d453b83
26 changed files with 290 additions and 75 deletions

4
Vagrantfile vendored
View File

@ -293,8 +293,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
end
(0..NNFSS - 1).each do |i|
config.vm.define "nfs#{i}" do |nfs|
nfs.vm.hostname = "#{LABEL_PREFIX}ceph-nfs#{i}"
config.vm.define "#{LABEL_PREFIX}nfs#{i}" do |nfs|
nfs.vm.hostname = "#{LABEL_PREFIX}nfs#{i}"
if ASSIGN_STATIC_IP
nfs.vm.network :private_network,
ip: "#{PUBLIC_SUBNET}.6#{i}"

View File

@ -138,6 +138,11 @@ dummy:
#ceph_stable_release: dummy
#ceph_stable_repo: "{{ ceph_mirror }}/debian-{{ ceph_stable_release }}"
#nfs_ganesha_stable: true # use stable repos for nfs-ganesha
#nfs_ganesha_stable_branch: V2.5-stable
#nfs_ganesha_stable_deb_repo: "{{ ceph_mirror }}/nfs-ganesha/deb-{{ nfs_ganesha_stable_branch }}/{{ ceph_stable_release }}"
# Use the option below to specify your applicable package tree, eg. when using non-LTS Ubuntu versions
# # for a list of available Debian distributions, visit http://download.ceph.com/debian-{{ ceph_stable_release }}/dists/
# for more info read: https://github.com/ceph/ceph-ansible/issues/305
@ -194,6 +199,11 @@ dummy:
#ceph_dev_branch: master # development branch you would like to use e.g: master, wip-hack
#ceph_dev_sha1: latest # distinct sha1 to use, defaults to 'latest' (as in latest built)
#nfs_ganesha_dev: false # use development repos for nfs-ganesha
# Set this to choose the version of ceph dev libraries used in the nfs-ganesha packages from shaman
# flavors so far include: ceph_master, ceph_jewel, ceph_kraken, ceph_luminous
#nfs_ganesha_flavor: "ceph_master"
# REPOSITORY: CUSTOM
@ -383,13 +393,17 @@ dummy:
#handler_health_rgw_check_retries: 5
#handler_health_rgw_check_delay: 10
###############
# NFS-GANESHA #
###############
# Confiure the type of NFS gatway access. At least one must be enabled for an
# NFS role to be useful
#
# Set this to true to enable File access via NFS. Requires an MDS role.
#nfs_file_gw: true
#nfs_file_gw: false
# Set this to true to enable Object access via NFS. Requires an RGW role.
#nfs_obj_gw: false
#nfs_obj_gw: true
###################
# CONFIG OVERRIDE #

View File

@ -15,14 +15,20 @@ dummy:
#fetch_directory: fetch/
# Even though NFS nodes should not have the admin key
# at their disposal, some people might want to have it
# distributed on RGW nodes. Setting 'copy_admin_key' to 'true'
# will copy the admin key to the /etc/ceph/ directory
#copy_admin_key: false
#######################
# Access type options #
#######################
# These are currently in ceph-common defaults because nfs_obj_gw shared with ceph-rgw
# Enable NFS File access
#nfs_file_gw: true
#nfs_file_gw: false
# Enable NFS Object access
#nfs_obj_gw: false
#nfs_obj_gw: true
######################
# NFS Ganesha Config #
@ -54,23 +60,27 @@ dummy:
#ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY"
#ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C"
###################
# CONFIG OVERRIDE #
###################
# Ganesha configuration file override.
# This allows you to specify more configuration options
# using an INI style format.
# The following sections are supported: [global], [mon], [osd], [mds], [rgw]
# These multiline strings will be appended to the contents of the blocks in ganesha.conf and
# must be in the correct ganesha.conf format seen here:
# https://github.com/nfs-ganesha/nfs-ganesha/blob/next/src/config_samples/ganesha.conf.example
#
# Example:
# ceph_conf_overrides:
# global:
# foo: 1234
# bar: 5678
#CACHEINODE {
# #Entries_HWMark = 100000;
#}
#
#ganesha_conf_overrides: {}
#ganesha_ceph_export_overrides:
#ganesha_rgw_export_overrides:
#ganesha_log_overrides:
#ganesha_conf_overrides: |
# CACHEINODE {
# #Entries_HWMark = 100000;
# }
##########
# DOCKER #

View File

@ -138,6 +138,11 @@ ceph_repository: rhcs
#ceph_stable_release: dummy
#ceph_stable_repo: "{{ ceph_mirror }}/debian-{{ ceph_stable_release }}"
#nfs_ganesha_stable: true # use stable repos for nfs-ganesha
#nfs_ganesha_stable_branch: V2.5-stable
#nfs_ganesha_stable_deb_repo: "{{ ceph_mirror }}/nfs-ganesha/deb-{{ nfs_ganesha_stable_branch }}/{{ ceph_stable_release }}"
# Use the option below to specify your applicable package tree, eg. when using non-LTS Ubuntu versions
# # for a list of available Debian distributions, visit http://download.ceph.com/debian-{{ ceph_stable_release }}/dists/
# for more info read: https://github.com/ceph/ceph-ansible/issues/305
@ -194,6 +199,11 @@ ceph_repository: rhcs
#ceph_dev_branch: master # development branch you would like to use e.g: master, wip-hack
#ceph_dev_sha1: latest # distinct sha1 to use, defaults to 'latest' (as in latest built)
#nfs_ganesha_dev: false # use development repos for nfs-ganesha
# Set this to choose the version of ceph dev libraries used in the nfs-ganesha packages from shaman
# flavors so far include: ceph_master, ceph_jewel, ceph_kraken, ceph_luminous
#nfs_ganesha_flavor: "ceph_master"
# REPOSITORY: CUSTOM
@ -383,13 +393,17 @@ ceph_repository: rhcs
#handler_health_rgw_check_retries: 5
#handler_health_rgw_check_delay: 10
###############
# NFS-GANESHA #
###############
# Confiure the type of NFS gatway access. At least one must be enabled for an
# NFS role to be useful
#
# Set this to true to enable File access via NFS. Requires an MDS role.
#nfs_file_gw: true
#nfs_file_gw: false
# Set this to true to enable Object access via NFS. Requires an RGW role.
#nfs_obj_gw: false
#nfs_obj_gw: true
###################
# CONFIG OVERRIDE #

View File

@ -10,14 +10,11 @@
state: present
changed_when: false
- name: configure debian gluster nfs ganesha community repository for rgw nfs
- name: add nfs-ganesha stable repository
apt_repository:
repo: "{{ item }}"
repo: "deb {{ nfs_ganesha_stable_deb_repo }} {{ ceph_stable_distro_source | default(ansible_lsb.codename) }} main"
state: present
with_items:
- ppa:gluster/libntirpc
- ppa:gluster/nfs-ganesha
changed_when: false
when:
- (nfs_obj_gw or nfs_file_gw)
- not ansible_distribution == "Debian"
- nfs_group_name in group_names
- nfs_ganesha_stable

View File

@ -10,3 +10,24 @@
repo: "{{ ceph_dev_deb_repo.content }}"
state: present
changed_when: false
- name: fetch nfs-ganesha development repository
uri:
url: https://shaman.ceph.com/api/repos/nfs-ganesha/next/latest/{{ ansible_distribution | lower }}/{{ ansible_distribution_release }}/flavors/{{ nfs_ganesha_flavor }}/repo
return_content: yes
register: nfs_ganesha_apt_repo
when:
- nfs_group_name in group_names
- nfs_ganesha_dev
- name: add nfs-ganesha development repository
copy:
content: "{{ nfs_ganesha_dev_apt_repo.content }}"
dest: /etc/apt/sources.list.d/nfs-ganesha-dev.list
owner: root
group: root
backup: yes
when:
- nfs_group_name in group_names
- nfs_ganesha_dev

View File

@ -43,10 +43,26 @@
- name: install red hat storage nfs gateway for debian
apt:
name: nfs-ganesha-fsal
name: nfs-ganesha
state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}"
when: nfs_group_name in group_names
- name: install red hat storage nfs file gateway
apt:
name: nfs-ganesha-ceph
state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}"
when:
- nfs_group_name in group_names
- nfs_file_gw
- name: install red hat storage nfs obj gateway
apt:
name: nfs-ganesha-rgw
state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}"
when:
- nfs_group_name in group_names
- nfs_obj_gw
- name: install ceph mgr for debian
apt:
pkg: ceph-mgr

View File

@ -44,13 +44,22 @@
when:
- nfs_group_name in group_names
- name: install nfs gateway for debian
apt:
name: nfs-ganesha-fsal
state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}"
update_cache: yes
- name: install nfs cephfs gateway
package:
name: nfs-ganesha-ceph
when:
- nfs_group_name in group_names
- nfs_file_gw
- name: install nfs rgw gateway
package:
name: "{{ item }}"
with_items:
- nfs-ganesha-rgw
- radosgw
when:
- nfs_group_name in group_names
- nfs_obj_gw
- name: install ceph mgr for debian
apt:

View File

@ -73,28 +73,22 @@
- nfs_group_name in group_names
- nfs_file_gw
- name: install redhat nfs-ganesha-rgw and ceph-radosgw packages
- name: install nfs cephfs gateway
package:
name: "{{ item }}"
with_items:
- nfs-ganesha-fsal
- ceph-radosgw
name: nfs-ganesha-ceph
when:
- nfs_group_name in group_names
- nfs_obj_gw
- ceph_rhcs_version == "2"
- nfs_file_gw
- name: install redhat nfs-ganesha-rgw and ceph-radosgw packages
package:
name: "{{ item }}"
with_items:
- nfs-ganesha-rgw
- nfs-ganesha-ceph
- ceph-radosgw
when:
- nfs_group_name in group_names
- nfs_obj_gw
- ceph_rhcs_version == "3"
- name: install redhat ceph-mgr package
package:

View File

@ -12,3 +12,16 @@
state: present
gpgkey: "{{ ceph_stable_key }}"
baseurl: "{{ ceph_mirror }}/rpm-{{ ceph_stable_release }}/{{ ceph_stable_redhat_distro }}/$basearch"
- name: add nfs-ganesha stable repository
yum_repository:
name: nfs_ganesha_stable
description: nfs-ganesha stable repo
gpgcheck: yes
state: present
gpgkey: "{{ ceph_stable_key }}"
baseurl: "{{ ceph_mirror }}/nfs-ganesha/rpm-{{ nfs_ganesha_stable_branch }}/{{ ceph_stable_release }}/$basearch"
when:
- nfs_group_name in group_names
- nfs_ganesha_stable

View File

@ -12,3 +12,24 @@
owner: root
group: root
backup: yes
- name: fetch nfs-ganesha red hat development repository
uri:
url: https://shaman.ceph.com/api/repos/nfs-ganesha/next/latest/{{ ansible_distribution | lower }}/{{ ansible_distribution_major_version }}/flavors/{{ nfs_ganesha_flavor }}/repo
return_content: yes
register: nfs_ganesha_dev_yum_repo
when:
- nfs_group_name in group_names
- nfs_ganesha_dev
- name: add nfs-ganesha development repository
copy:
content: "{{ nfs_ganesha_dev_yum_repo.content }}"
dest: /etc/yum.repos.d/nfs-ganesha-dev.repo
owner: root
group: root
backup: yes
when:
- nfs_group_name in group_names
- nfs_ganesha_dev

View File

@ -130,6 +130,11 @@ ceph_stable_key: https://download.ceph.com/keys/release.asc
ceph_stable_release: dummy
ceph_stable_repo: "{{ ceph_mirror }}/debian-{{ ceph_stable_release }}"
nfs_ganesha_stable: true # use stable repos for nfs-ganesha
nfs_ganesha_stable_branch: V2.5-stable
nfs_ganesha_stable_deb_repo: "{{ ceph_mirror }}/nfs-ganesha/deb-{{ nfs_ganesha_stable_branch }}/{{ ceph_stable_release }}"
# Use the option below to specify your applicable package tree, eg. when using non-LTS Ubuntu versions
# # for a list of available Debian distributions, visit http://download.ceph.com/debian-{{ ceph_stable_release }}/dists/
# for more info read: https://github.com/ceph/ceph-ansible/issues/305
@ -186,6 +191,11 @@ ceph_rhcs_repository_path: "{{ ceph_stable_rh_storage_repository_path | default(
ceph_dev_branch: master # development branch you would like to use e.g: master, wip-hack
ceph_dev_sha1: latest # distinct sha1 to use, defaults to 'latest' (as in latest built)
nfs_ganesha_dev: false # use development repos for nfs-ganesha
# Set this to choose the version of ceph dev libraries used in the nfs-ganesha packages from shaman
# flavors so far include: ceph_master, ceph_jewel, ceph_kraken, ceph_luminous
nfs_ganesha_flavor: "ceph_master"
# REPOSITORY: CUSTOM
@ -375,13 +385,17 @@ handler_health_mds_check_delay: 10
handler_health_rgw_check_retries: 5
handler_health_rgw_check_delay: 10
###############
# NFS-GANESHA #
###############
# Confiure the type of NFS gatway access. At least one must be enabled for an
# NFS role to be useful
#
# Set this to true to enable File access via NFS. Requires an MDS role.
nfs_file_gw: true
nfs_file_gw: false
# Set this to true to enable Object access via NFS. Requires an RGW role.
nfs_obj_gw: false
nfs_obj_gw: true
###################
# CONFIG OVERRIDE #

View File

@ -7,14 +7,20 @@
fetch_directory: fetch/
# Even though NFS nodes should not have the admin key
# at their disposal, some people might want to have it
# distributed on RGW nodes. Setting 'copy_admin_key' to 'true'
# will copy the admin key to the /etc/ceph/ directory
copy_admin_key: false
#######################
# Access type options #
#######################
# These are currently in ceph-common defaults because nfs_obj_gw shared with ceph-rgw
# Enable NFS File access
#nfs_file_gw: true
nfs_file_gw: false
# Enable NFS Object access
#nfs_obj_gw: false
nfs_obj_gw: true
######################
# NFS Ganesha Config #
@ -46,23 +52,27 @@ ceph_nfs_rgw_user: "cephnfs"
#ceph_nfs_rgw_access_key: "QFAMEDSJP5DEKJO0DDXY"
#ceph_nfs_rgw_secret_key: "iaSFLDVvDdQt6lkNzHyW4fPLZugBAI1g17LO0+87[MAC[M#C"
###################
# CONFIG OVERRIDE #
###################
# Ganesha configuration file override.
# This allows you to specify more configuration options
# using an INI style format.
# The following sections are supported: [global], [mon], [osd], [mds], [rgw]
# These multiline strings will be appended to the contents of the blocks in ganesha.conf and
# must be in the correct ganesha.conf format seen here:
# https://github.com/nfs-ganesha/nfs-ganesha/blob/next/src/config_samples/ganesha.conf.example
#
# Example:
# ceph_conf_overrides:
# global:
# foo: 1234
# bar: 5678
#CACHEINODE {
#Entries_HWMark = 100000;
#}
#
ganesha_conf_overrides: {}
#ganesha_ceph_export_overrides:
#ganesha_rgw_export_overrides:
#ganesha_log_overrides:
#ganesha_conf_overrides: |
# CACHEINODE {
#Entries_HWMark = 100000;
# }
##########
# DOCKER #

View File

@ -1,4 +1,26 @@
---
- name: create rgw nfs user
command: radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name="RGW NFS User"
register: rgwuser
delegate_to: "{{ groups[mon_group_name][0] }}"
when: nfs_obj_gw
- name: set access key
set_fact:
ceph_nfs_rgw_access_key: "{{ (rgwuser.stdout | from_json)['keys'][0]['access_key'] }}"
delegate_to: "{{ groups[mon_group_name][0] }}"
when:
- nfs_obj_gw
- ceph_nfs_rgw_access_key is not defined
- name: set secret key
set_fact:
ceph_nfs_rgw_secret_key: "{{(rgwuser.stdout | from_json)['keys'][0]['secret_key']}}"
delegate_to: "{{ groups[mon_group_name][0] }}"
when:
- nfs_obj_gw
- ceph_nfs_rgw_secret_key is not defined
- name: create nfs gateway directories
file:
path: "{{ item }}"
@ -10,31 +32,17 @@
- /var/lib/nfs/ganesha
- /var/run/ganesha
- name: create rgw nfs user
command: radosgw-admin --cluster {{ cluster }} user create --uid={{ ceph_nfs_rgw_user }} --display-name="RGW NFS User"
register: rgwuser
when: nfs_obj_gw
- name: set access key
set_fact:
ceph_nfs_rgw_access_key: "{{ (rgwuser.stdout | from_json)['keys'][0]['access_key'] }}"
when: nfs_obj_gw
- name: set secret key
set_fact:
ceph_nfs_rgw_secret_key: "{{(rgwuser.stdout | from_json)['keys'][0]['secret_key']}}"
when: nfs_obj_gw
- name: generate ganesha configuration file
action: config_template
args:
src: "{{ lookup('env', 'ANSIBLE_ROLES_PATH') | default (playbook_dir + '/roles', true) }}/ceph-common/templates/ganesha.conf.j2"
src: "{{ lookup('env', 'ANSIBLE_ROLES_PATH') | default (playbook_dir + '/roles', true) }}/ceph-nfs/templates/ganesha.conf.j2"
dest: /etc/ganesha/ganesha.conf
owner: "root"
group: "root"
mode: "0644"
config_overrides: "{{ ganesha_conf_overrides }}"
config_type: ini
notify:
- restart ceph nfss
- name: start nfs gateway service
service:

View File

@ -12,15 +12,17 @@ EXPORT
Access_Type = {{ ceph_nfs_ceph_access_type }};
NFS_Protocols = {{ ceph_nfs_ceph_protocols }};
Protocols = {{ ceph_nfs_ceph_protocols }};
Transport_Protocols = TCP;
Transports = TCP;
Sectype = sys,krb5,krb5i,krb5p;
FSAL {
Name = CEPH;
}
{{ ganesha_ceph_export_overrides | default(None) }}
}
{% endif %}
{% if nfs_obj_gw %}
@ -34,9 +36,9 @@ EXPORT
Access_Type = {{ ceph_nfs_rgw_access_type }};
NFS_Protocols = {{ ceph_nfs_rgw_protocols }};
Protocols = {{ ceph_nfs_rgw_protocols }};
Transport_Protocols = TCP;
Transports = TCP;
Sectype = sys,krb5,krb5i,krb5p;
@ -46,6 +48,9 @@ EXPORT
Access_Key_Id ="{{ ceph_nfs_rgw_access_key }}";
Secret_Access_Key = "{{ ceph_nfs_rgw_secret_key }}";
}
{{ ganesha_rgw_export_overrides | default(None) }}
}
{% endif %}
@ -55,4 +60,8 @@ LOG {
destination = "{{ ceph_nfs_log_file }}";
enable = active;
}
{{ ganesha_log_overrides | default(None) }}
}
{{ ganesha_conf_overrides | default(None) }}

View File

@ -103,6 +103,8 @@ def pytest_collection_modifyitems(session, config, items):
item.add_marker(pytest.mark.mdss)
elif "rgw" in test_path:
item.add_marker(pytest.mark.rgws)
elif "nfs" in test_path:
item.add_marker(pytest.mark.nfss)
else:
item.add_marker(pytest.mark.all)

View File

@ -22,3 +22,7 @@ ceph_conf_overrides:
global:
osd_pool_default_pg_num: 8
osd_pool_default_size: 1
nfs_ganesha_stable: true
nfs_ganesha_dev: false
nfs_ganesha_stable_branch: V2.5-stable
nfs_ganesha_flavor: "ceph_master"

View File

@ -0,0 +1,7 @@
copy_admin_key: true
nfs_file_gw: false
nfs_obj_gw: true
ganesha_conf_overrides: |
CACHEINODE {
Entries_HWMark = 100000;
}

View File

@ -17,3 +17,6 @@ ceph-client0
[mgrs]
ceph-mgr0
[nfss]
ceph-nfs0

View File

@ -8,7 +8,7 @@ mon_vms: 3
osd_vms: 1
mds_vms: 1
rgw_vms: 1
nfs_vms: 0
nfs_vms: 1
rbd_mirror_vms: 0
client_vms: 1
iscsi_gw_vms: 0

View File

@ -13,5 +13,16 @@
dest: "{{ change_dir }}/group_vars/all"
when: change_dir is defined
- name: change nfs-ganesha repository to 'dev'
replace:
regexp: "nfs_ganesha_stable:.*"
replace: "nfs_ganesha_stable: false"
dest: "{{ change_dir }}/group_vars/all"
replace:
regexp: "nfs_ganesha_dev:.*"
replace: "nfs_ganesha_dev: true"
dest: "{{ change_dir }}/group_vars/all"
when: change_dir is defined
- name: print contents of {{ change_dir }}/group_vars/all
command: "cat {{ change_dir }}/group_vars/all"

View File

@ -0,0 +1,23 @@
import pytest
class TestNFSs(object):
@pytest.mark.no_docker
def test_nfs_ganesha_is_installed(self, node, host):
assert host.package("nfs-ganesha").is_installed
@pytest.mark.no_docker
def test_nfs_ganesha_rgw_package_is_installed(self, node, host):
assert host.package("nfs-ganesha-rgw").is_installed
@pytest.mark.no_docker
def test_nfs_services_are_running(self, node, host):
assert host.service("nfs-ganesha").is_running
@pytest.mark.no_docker
def test_nfs_services_are_enabled(self, node, host):
assert host.service("nfs-ganesha").is_enabled
@pytest.mark.no_docker
def test_nfs_config_override(self, node, host):
assert host.file("/etc/ganesha/ganesha.conf").contains("Entries_HWMark")

View File

@ -24,3 +24,8 @@ debian_ceph_packages:
- ceph
- ceph-common
- ceph-fuse
nfs_ganesha_stable: true
nfs_ganesha_stable_branch: V2.5-stable
nfs_ganesha_stable_deb_repo: "{{ ceph_mirror }}/nfs-ganesha/deb-{{ nfs_ganesha_stable_branch }}/{{ ceph_stable_release }}"
nfs_ganesha_dev: false
nfs_ganesha_flavor: "ceph_master"

View File

@ -0,0 +1,7 @@
copy_admin_key: true
nfs_file_gw: false
nfs_obj_gw: true
ganesha_conf_overrides: |
CACHEINODE {
Entries_HWMark = 100000;
}

View File

@ -17,3 +17,6 @@ client0
[mgrs]
mgr0
[nfss]
nfs0

View File

@ -8,7 +8,7 @@ mon_vms: 3
osd_vms: 1
mds_vms: 1
rgw_vms: 1
nfs_vms: 0
nfs_vms: 1
rbd_mirror_vms: 0
client_vms: 1
iscsi_gw_vms: 0