From 3624a85b64fdad391c8eb582160c515a28dd6fe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 20 Jan 2015 18:43:47 +0100 Subject: [PATCH] Docker support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WIP! Signed-off-by: Sébastien Han --- README.md | 6 +++ roles/ceph-common/defaults/main.yml | 25 ++++++++----- roles/ceph-mds/defaults/main.yml | 9 +++++ roles/ceph-mds/tasks/docker.yml | 29 +++++++++++++++ roles/ceph-mds/tasks/main.yml | 42 ++------------------- roles/ceph-mds/tasks/pre_requisite.yml | 40 ++++++++++++++++++++ roles/ceph-mon/defaults/main.yml | 21 +++++++++-- roles/ceph-mon/tasks/docker.yml | 42 +++++++++++++++++++++ roles/ceph-mon/tasks/main.yml | 9 +++++ roles/ceph-osd/defaults/main.yml | 9 +++++ roles/ceph-osd/tasks/docker.yml | 29 +++++++++++++++ roles/ceph-osd/tasks/main.yml | 32 ++++------------ roles/ceph-osd/tasks/pre_requisite.yml | 23 ++++++++++++ roles/ceph-radosgw/defaults/main.yml | 11 +++++- roles/ceph-radosgw/tasks/docker.yml | 29 +++++++++++++++ roles/ceph-radosgw/tasks/main.yml | 43 ++++------------------ roles/ceph-radosgw/tasks/pre_requisite.yml | 25 +++++++++++++ roles/ceph-radosgw/tasks/start_radosgw.yml | 10 +++++ site.yml | 2 +- 19 files changed, 324 insertions(+), 112 deletions(-) create mode 100644 roles/ceph-mds/tasks/docker.yml create mode 100644 roles/ceph-mds/tasks/pre_requisite.yml create mode 100644 roles/ceph-mon/tasks/docker.yml create mode 100644 roles/ceph-osd/tasks/docker.yml create mode 100644 roles/ceph-osd/tasks/pre_requisite.yml create mode 100644 roles/ceph-radosgw/tasks/docker.yml create mode 100644 roles/ceph-radosgw/tasks/pre_requisite.yml create mode 100644 roles/ceph-radosgw/tasks/start_radosgw.yml diff --git a/README.md b/README.md index d641e9e25..88b7c2017 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,12 @@ ceph-ansible Ansible playbook for Ceph! +Clone me: + +```bash +git clone https://github.com/ceph/ceph-ansible.git +``` + ## What does it do? General support for: diff --git a/roles/ceph-common/defaults/main.yml b/roles/ceph-common/defaults/main.yml index e7c8048e7..a0ab1bc2f 100644 --- a/roles/ceph-common/defaults/main.yml +++ b/roles/ceph-common/defaults/main.yml @@ -1,9 +1,9 @@ --- # You can override vars by using host or group vars -######### -# INSTALL -######### +########### +# INSTALL # +########### # /!\ EITHER ACTIVE ceph_stable OR ceph_stable_ice OR ceph_dev /!\ @@ -43,9 +43,9 @@ ceph_dev_branch: master # development branch you would like to use e.g: master, ceph_dev_redhat_distro: centos7 -#################### -# CEPH CONFIGURATION -#################### +###################### +# CEPH CONFIGURATION # +###################### ## Ceph options # @@ -134,9 +134,9 @@ radosgw_civetweb_port: 80 #common_single_host_mode: true -########### -# OS TUNING -########### +############# +# OS TUNING # +############# disable_transparent_hugepage: true disable_swap: true @@ -145,3 +145,10 @@ os_tuning_params: - { name: fs.file-max, value: 26234859 } - { name: vm.zone_reclaim_mode, value: 0 } - { name: vm.vfs_cache_pressure, value: 50 } + + +########## +# DOCKER # +########## + +docker: false diff --git a/roles/ceph-mds/defaults/main.yml b/roles/ceph-mds/defaults/main.yml index bd46d3bd5..6734ef275 100644 --- a/roles/ceph-mds/defaults/main.yml +++ b/roles/ceph-mds/defaults/main.yml @@ -2,3 +2,12 @@ # You can override vars by using host or group vars cephx: true + + +########## +# DOCKER # +########## + +ceph_containerized_deployment: false +ceph_mds_docker_username: ceph +ceph_mds_docker_imagename: "mds:latest" diff --git a/roles/ceph-mds/tasks/docker.yml b/roles/ceph-mds/tasks/docker.yml new file mode 100644 index 000000000..e5d674da7 --- /dev/null +++ b/roles/ceph-mds/tasks/docker.yml @@ -0,0 +1,29 @@ +--- +- name: Fetch Ceph config and keys + copy: > + src=fetch/docker_mon_files/{{ item }} + dest=/etc/ceph/ + owner=root + group=root + mode=600 + with_items: + - /etc/ceph/ceph.client.admin.keyring + - /etc/ceph/ceph.conf + - /etc/ceph/monmap + - /etc/ceph/ceph.mon.keyring + +- name: Run the Ceph Medata docker image + docker: > + image="{{ ceph_mds_docker_username }}/{{ ceph_mds_docker_imagename }}" + name=ceph-mds-{{ ansible_hostname }} + net=host + state=running + env="MDS_NAME=ceph-mds-{{ ansible_hostname }}, MDS_CIVETWEB_PORT={{ ceph_mds_civetweb_port }}" + volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" + +- name: Ensure ceph_mds service is running + docker: > + image="{{ ceph_mds_docker_username }}/{{ ceph_mds_docker_imagename }}" + name="ceph-{{ ansible_hostname }}" + detach=yes + state=running diff --git a/roles/ceph-mds/tasks/main.yml b/roles/ceph-mds/tasks/main.yml index 35393ba42..fcdbce94e 100644 --- a/roles/ceph-mds/tasks/main.yml +++ b/roles/ceph-mds/tasks/main.yml @@ -1,40 +1,6 @@ --- -## Deploy Ceph metadata server(s) +- include: pre_requisite.yml + when: not ceph_containerized_deployment -- name: Copy MDS bootstrap key - copy: > - src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-mds/ceph.keyring - dest=/var/lib/ceph/bootstrap-mds/ceph.keyring - owner=root - group=root - mode=600 - when: cephx - -- name: Create MDS directory - file: > - path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }} - state=directory - owner=root - group=root - mode=0644 - when: cephx - -- name: Create MDS keyring - command: ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.{{ ansible_hostname }} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring creates=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring - when: cephx - changed_when: False - -- name: Set MDS key permissions - file: > - path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring - mode=0600 - owner=root - group=root - when: cephx - -- name: Start and add that the MDS service to the init sequence - service: > - name=ceph - state=started - enabled=yes - args=mds +- include: docker.yml + when: ceph_containerized_deployment diff --git a/roles/ceph-mds/tasks/pre_requisite.yml b/roles/ceph-mds/tasks/pre_requisite.yml new file mode 100644 index 000000000..bc83aede1 --- /dev/null +++ b/roles/ceph-mds/tasks/pre_requisite.yml @@ -0,0 +1,40 @@ +--- +# Deploy Ceph metadata server(s) + +- name: Copy MDS bootstrap key + copy: > + src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-mds/ceph.keyring + dest=/var/lib/ceph/bootstrap-mds/ceph.keyring + owner=root + group=root + mode=600 + when: cephx + +- name: Create MDS directory + file: > + path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }} + state=directory + owner=root + group=root + mode=0644 + when: cephx + +- name: Create MDS keyring + command: ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.{{ ansible_hostname }} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring creates=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring + when: cephx + changed_when: False + +- name: Set MDS key permissions + file: > + path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring + mode=0600 + owner=root + group=root + when: cephx + +- name: Start and add that the MDS service to the init sequence + service: > + name=ceph + state=started + enabled=yes + args=mds diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml index c26322150..74728971d 100644 --- a/roles/ceph-mon/defaults/main.yml +++ b/roles/ceph-mon/defaults/main.yml @@ -1,6 +1,10 @@ --- # You can override vars by using host or group vars +########### +# GENERAL # +########### + # ACTIVATE BOTH FSID AND MONITOR_SECRET VARIABLES FOR NON-VAGRANT DEPLOYMENT fsid: "{{ cluster_uuid.stdout }}" # monitor_secret: @@ -10,9 +14,10 @@ cephx: true # referenced in common role too. radosgw: false -########### -# OPENSTACK -########### + +############# +# OPENSTACK # +############# openstack_config: false openstack_glance_pool: images @@ -24,3 +29,13 @@ openstack_keys: - { name: client.glance, value: "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ openstack_glance_pool }}'" } - { name: client.cinder, value: "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ openstack_cinder_pool }}, allow rwx pool={{ openstack_nova_pool }}, allow rx pool={{ openstack_glance_pool }}'" } - { name: client.cinder-backup, value: "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ openstack_cinder_backup_pool }}'" } + + +########## +# DOCKER # +########## + +ceph_containerized_deployment: false +ceph_mon_docker_interface: eth0 +ceph_mon_docker_username: ceph +ceph_mon_docker_imagename: "mon:latest" diff --git a/roles/ceph-mon/tasks/docker.yml b/roles/ceph-mon/tasks/docker.yml new file mode 100644 index 000000000..1f3427f1d --- /dev/null +++ b/roles/ceph-mon/tasks/docker.yml @@ -0,0 +1,42 @@ +--- +- name: Try to fetch Ceph config and keys + copy: > + src=fetch/docker_mon_files/{{ item }} + dest=/etc/ceph/ + owner=root + group=root + mode=600 + with_items: + - /etc/ceph/ceph.client.admin.keyring + - /etc/ceph/ceph.conf + - /etc/ceph/monmap + - /etc/ceph/ceph.mon.keyring + ignore_errors: true + +- name: Run the Ceph Monitor docker image + docker: > + image="{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" + name=ceph-{{ ansible_hostname }} + net=host + state=running + env="MON_NAME=ceph-{{ ansible_hostname }},MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }}" + volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" + +- name: Ensure ceph_mon service is running + docker: > + image="{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" + name="ceph-{{ ansible_hostname }}" + ports=6789:6789 + detach=yes + state=running + +- name: Collect Ceph files to the Ansible server + fetch: > + src={{ item }} + dest=fetch/docker_mon_files/{{ item }} + flat=yes + with_items: + - /etc/ceph/ceph.client.admin.keyring + - /etc/ceph/ceph.conf + - /etc/ceph/monmap + - /etc/ceph/ceph.mon.keyring diff --git a/roles/ceph-mon/tasks/main.yml b/roles/ceph-mon/tasks/main.yml index 7077a7def..0a7415374 100644 --- a/roles/ceph-mon/tasks/main.yml +++ b/roles/ceph-mon/tasks/main.yml @@ -1,3 +1,12 @@ --- - include: deploy_monitors.yml + when: not ceph_containerized_deployment + - include: ceph_keys.yml + when: not ceph_containerized_deployment + +- include: ceph_keys.yml + when: not ceph_containerized_deployment + +- include: docker.yml + when: ceph_containerized_deployment diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml index f6cc08a00..5f767c2a1 100644 --- a/roles/ceph-osd/defaults/main.yml +++ b/roles/ceph-osd/defaults/main.yml @@ -104,3 +104,12 @@ osd_directories: - /var/lib/ceph/osd/mydir2 - /var/lib/ceph/osd/mydir3 - /var/lib/ceph/osd/mydir4 + + +########## +# DOCKER # +########## + +ceph_containerized_deployment: false +ceph_osd_docker_username: ceph +ceph_osd_docker_imagename: "osd:latest" diff --git a/roles/ceph-osd/tasks/docker.yml b/roles/ceph-osd/tasks/docker.yml new file mode 100644 index 000000000..9e6b7df22 --- /dev/null +++ b/roles/ceph-osd/tasks/docker.yml @@ -0,0 +1,29 @@ +--- +- name: Fetch Ceph config and keys + copy: > + src=fetch/docker_mon_files/{{ item }} + dest=/etc/ceph/ + owner=root + group=root + mode=600 + with_items: + - /etc/ceph/ceph.client.admin.keyring + - /etc/ceph/ceph.conf + - /etc/ceph/monmap + - /etc/ceph/ceph.mon.keyring + +- name: Run the Ceph OSD docker image + docker: > + image="{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" + name=ceph-{{ ansible_hostname }} + net=host + state=running + env="OSD_NAME=ceph-{{ ansible_hostname }}" + volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" + +- name: Ensure ceph_osd service is running + docker: > + image="{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" + name="ceph-{{ ansible_hostname }}" + detach=yes + state=running diff --git a/roles/ceph-osd/tasks/main.yml b/roles/ceph-osd/tasks/main.yml index e7bc3d413..fca779bb5 100644 --- a/roles/ceph-osd/tasks/main.yml +++ b/roles/ceph-osd/tasks/main.yml @@ -1,32 +1,16 @@ --- -## Deploy Ceph Oject Storage Daemon(s) -- name: Install dependencies - apt: > - pkg=parted - state=present - when: ansible_os_family == 'Debian' - -- name: Install dependencies - yum: > - name=parted - state=present - when: ansible_os_family == 'RedHat' - -- name: Copy OSD bootstrap key - copy: > - src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-osd/ceph.keyring - dest=/var/lib/ceph/bootstrap-osd/ceph.keyring - owner=root - group=root - mode=600 - when: cephx +- include: pre_requisite.yml + when: not ceph_containerized_deployment - include: journal_collocation.yml - when: journal_collocation + when: journal_collocation and not ceph_containerized_deployment - include: raw_multi_journal.yml - when: raw_multi_journal + when: raw_multi_journal and not ceph_containerized_deployment - include: osd_directory.yml - when: osd_directory + when: osd_directory and not ceph_containerized_deployment + +- include: docker.yml + when: ceph_containerized_deployment diff --git a/roles/ceph-osd/tasks/pre_requisite.yml b/roles/ceph-osd/tasks/pre_requisite.yml new file mode 100644 index 000000000..c0cd8d6d1 --- /dev/null +++ b/roles/ceph-osd/tasks/pre_requisite.yml @@ -0,0 +1,23 @@ +--- +## Deploy Ceph Oject Storage Daemon(s) + +- name: Install dependencies + apt: > + pkg=parted + state=present + when: ansible_os_family == 'Debian' + +- name: Install dependencies + yum: > + name=parted + state=present + when: ansible_os_family == 'RedHat' + +- name: Copy OSD bootstrap key + copy: > + src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-osd/ceph.keyring + dest=/var/lib/ceph/bootstrap-osd/ceph.keyring + owner=root + group=root + mode=600 + when: cephx diff --git a/roles/ceph-radosgw/defaults/main.yml b/roles/ceph-radosgw/defaults/main.yml index 91f1a4c64..241b2a83c 100644 --- a/roles/ceph-radosgw/defaults/main.yml +++ b/roles/ceph-radosgw/defaults/main.yml @@ -12,5 +12,14 @@ http_100_continue: false # Rados Gateway options redhat_distro_ceph_extra: centos6.4 # supported distros are centos6.3, centos6.4, centos6, fedora18, fedora19, opensuse12.2, rhel6.3, rhel6.4, rhel6.5, rhel6, sles11sp2 email_address: foo@bar.com - radosgw_frontend: civetweb # we currently only support a single backend + + +########## +# DOCKER # +########## + +ceph_containerized_deployment: false +ceph_rgw_civetweb_port: 80 +ceph_rgw_docker_username: ceph +ceph_rgw_docker_imagename: "rgw:latest" diff --git a/roles/ceph-radosgw/tasks/docker.yml b/roles/ceph-radosgw/tasks/docker.yml new file mode 100644 index 000000000..51438afb7 --- /dev/null +++ b/roles/ceph-radosgw/tasks/docker.yml @@ -0,0 +1,29 @@ +--- +- name: Fetch Ceph config and keys + copy: > + src=fetch/docker_mon_files/{{ item }} + dest=/etc/ceph/ + owner=root + group=root + mode=600 + with_items: + - /etc/ceph/ceph.client.admin.keyring + - /etc/ceph/ceph.conf + - /etc/ceph/monmap + - /etc/ceph/ceph.mon.keyring + +- name: Run the Ceph Monitor docker image + docker: > + image="{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}" + name=ceph-rgw-{{ ansible_hostname }} + net=host + state=running + env="RGW_NAME=ceph-rgw-{{ ansible_hostname }}, RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" + volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" + +- name: Ensure ceph_rgw service is running + docker: > + image="{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}" + name="ceph-{{ ansible_hostname }}" + detach=yes + state=running diff --git a/roles/ceph-radosgw/tasks/main.yml b/roles/ceph-radosgw/tasks/main.yml index 583609492..0d7aecb50 100644 --- a/roles/ceph-radosgw/tasks/main.yml +++ b/roles/ceph-radosgw/tasks/main.yml @@ -1,31 +1,7 @@ --- -- name: Copy RGW bootstrap key - copy: > - src=fetch/{{ fsid }}/etc/ceph/radosgw.{{ ansible_hostname }}.keyring - dest=/etc/ceph/radosgw.{{ ansible_hostname }}.keyring - owner=root - group=root - mode=600 - when: cephx -- name: Set RGW bootstrap key permissions - file: > - path=/etc/ceph/radosgw.{{ ansible_hostname }}.keyring - mode=0600 - owner=root - group=root - when: cephx - -- name: Create RGW directory - file: > - path=/var/lib/ceph/radosgw/{{ ansible_fqdn }} - state=directory - owner=root - group=root - mode=0644 - -## Check OS family -# +- include: pre_requisite.yml + when: not ceph_containerized_deployment - include: install_redhat.yml when: ansible_os_family == 'RedHat' and radosgw_frontend == 'apache' @@ -38,20 +14,15 @@ pkg=radosgw state=present update_cache=yes - when: ansible_os_family == 'Debian' + when: ansible_os_family == 'Debian' and not ceph_containerized_deployment - name: "Install Rados Gateway" yum: > name=ceph-radosgw state=present - when: ansible_os_family == 'RedHat' + when: ansible_os_family == 'RedHat' and not ceph_containerized_deployment -## If we don't perform this check Ansible will start multiple instance of radosgw -- name: Check if RGW is started - command: /etc/init.d/radosgw status - register: rgwstatus - ignore_errors: True +- include: start_radosgw.yml -- name: Start RGW - command: /etc/init.d/radosgw start - when: rgwstatus.rc != 0 +- include: docker.yml + when: ceph_containerized_deployment diff --git a/roles/ceph-radosgw/tasks/pre_requisite.yml b/roles/ceph-radosgw/tasks/pre_requisite.yml new file mode 100644 index 000000000..4e6415323 --- /dev/null +++ b/roles/ceph-radosgw/tasks/pre_requisite.yml @@ -0,0 +1,25 @@ +--- +- name: Copy RGW bootstrap key + copy: > + src=fetch/{{ fsid }}/etc/ceph/radosgw.{{ ansible_hostname }}.keyring + dest=/etc/ceph/radosgw.{{ ansible_hostname }}.keyring + owner=root + group=root + mode=600 + when: cephx + +- name: Set RGW bootstrap key permissions + file: > + path=/etc/ceph/radosgw.{{ ansible_hostname }}.keyring + mode=0600 + owner=root + group=root + when: cephx + +- name: Create RGW directory + file: > + path=/var/lib/ceph/radosgw/{{ ansible_fqdn }} + state=directory + owner=root + group=root + mode=0644 diff --git a/roles/ceph-radosgw/tasks/start_radosgw.yml b/roles/ceph-radosgw/tasks/start_radosgw.yml new file mode 100644 index 000000000..574b40862 --- /dev/null +++ b/roles/ceph-radosgw/tasks/start_radosgw.yml @@ -0,0 +1,10 @@ +--- +## If we don't perform this check Ansible will start multiple instance of radosgw +- name: Check if RGW is started + command: /etc/init.d/radosgw status + register: rgwstatus + ignore_errors: True + +- name: Start RGW + command: /etc/init.d/radosgw start + when: rgwstatus.rc != 0 diff --git a/site.yml b/site.yml index 2cded5238..90469c212 100644 --- a/site.yml +++ b/site.yml @@ -8,7 +8,7 @@ - rgws sudo: True roles: - - ceph-common + - { role: ceph-common, when: not docker } - hosts: mons sudo: True