Docker support

WIP!

Signed-off-by: Sébastien Han <sebastien.han@enovance.com>

README.md

@@ -3,6 +3,12 @@ ceph-ansible
 
 Ansible playbook for Ceph!
 
+Clone me:
+
+```bash
+git clone https://github.com/ceph/ceph-ansible.git
+```
+
 ## What does it do?
 
 General support for:

roles/ceph-common/defaults/main.yml

@@ -1,9 +1,9 @@
 ---
 # You can override vars by using host or group vars
 
-#########
-# INSTALL
-#########
+###########
+# INSTALL #
+###########
 
 # /!\ EITHER ACTIVE ceph_stable OR ceph_stable_ice OR ceph_dev /!\
 
@@ -43,9 +43,9 @@ ceph_dev_branch: master # development branch you would like to use e.g: master,
 ceph_dev_redhat_distro: centos7
 
 
-####################
-# CEPH CONFIGURATION
-####################
+######################
+# CEPH CONFIGURATION #
+######################
 
 ## Ceph options
 #
@@ -134,9 +134,9 @@ radosgw_civetweb_port: 80
 #common_single_host_mode: true
 
 
-###########
-# OS TUNING
-###########
+#############
+# OS TUNING #
+#############
 
 disable_transparent_hugepage: true
 disable_swap: true
@@ -145,3 +145,10 @@ os_tuning_params:
   - { name: fs.file-max, value: 26234859 }
   - { name: vm.zone_reclaim_mode, value: 0 }
   - { name: vm.vfs_cache_pressure, value: 50 }
+
+
+##########
+# DOCKER #
+##########
+
+docker: false

roles/ceph-mds/defaults/main.yml

@@ -2,3 +2,12 @@
 # You can override vars by using host or group vars
 
 cephx: true
+
+
+##########
+# DOCKER #
+##########
+
+ceph_containerized_deployment: false
+ceph_mds_docker_username: ceph
+ceph_mds_docker_imagename: "mds:latest"

roles/ceph-mds/tasks/docker.yml

@@ -0,0 +1,29 @@
+---
+- name: Fetch Ceph config and keys
+  copy: >
+    src=fetch/docker_mon_files/{{ item }}
+    dest=/etc/ceph/
+    owner=root
+    group=root
+    mode=600
+  with_items:
+    - /etc/ceph/ceph.client.admin.keyring
+    - /etc/ceph/ceph.conf
+    - /etc/ceph/monmap
+    - /etc/ceph/ceph.mon.keyring
+
+- name: Run the Ceph Medata docker image
+  docker: >
+    image="{{ ceph_mds_docker_username }}/{{ ceph_mds_docker_imagename }}"
+    name=ceph-mds-{{ ansible_hostname }}
+    net=host
+    state=running
+    env="MDS_NAME=ceph-mds-{{ ansible_hostname }}, MDS_CIVETWEB_PORT={{ ceph_mds_civetweb_port }}"
+    volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
+
+- name: Ensure ceph_mds service is running
+  docker: >
+    image="{{ ceph_mds_docker_username }}/{{ ceph_mds_docker_imagename }}"
+    name="ceph-{{ ansible_hostname }}"
+    detach=yes
+    state=running

roles/ceph-mds/tasks/main.yml

@@ -1,40 +1,6 @@
 ---
-## Deploy Ceph metadata server(s)
+- include: pre_requisite.yml
+  when: not ceph_containerized_deployment
 
-- name: Copy MDS bootstrap key
-  copy: >
-    src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-mds/ceph.keyring
-    dest=/var/lib/ceph/bootstrap-mds/ceph.keyring
-    owner=root
-    group=root
-    mode=600
-  when: cephx
-
-- name: Create MDS directory
-  file: >
-    path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}
-    state=directory
-    owner=root
-    group=root
-    mode=0644
-  when: cephx
-
-- name: Create MDS keyring
-  command: ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.{{ ansible_hostname }} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring creates=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring
-  when: cephx
-  changed_when: False
-
-- name: Set MDS key permissions
-  file: >
-    path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring
-    mode=0600
-    owner=root
-    group=root
-  when: cephx
-
-- name: Start and add that the MDS service to the init sequence
-  service: >
-    name=ceph
-    state=started
-    enabled=yes
-    args=mds
+- include: docker.yml
+  when: ceph_containerized_deployment

roles/ceph-mds/tasks/pre_requisite.yml

@@ -0,0 +1,40 @@
+---
+# Deploy Ceph metadata server(s)
+
+- name: Copy MDS bootstrap key
+  copy: >
+    src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-mds/ceph.keyring
+    dest=/var/lib/ceph/bootstrap-mds/ceph.keyring
+    owner=root
+    group=root
+    mode=600
+  when: cephx
+
+- name: Create MDS directory
+  file: >
+    path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}
+    state=directory
+    owner=root
+    group=root
+    mode=0644
+  when: cephx
+
+- name: Create MDS keyring
+  command: ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.{{ ansible_hostname }} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring creates=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring
+  when: cephx
+  changed_when: False
+
+- name: Set MDS key permissions
+  file: >
+    path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring
+    mode=0600
+    owner=root
+    group=root
+  when: cephx
+
+- name: Start and add that the MDS service to the init sequence
+  service: >
+    name=ceph
+    state=started
+    enabled=yes
+    args=mds

roles/ceph-mon/defaults/main.yml

@@ -1,6 +1,10 @@
 ---
 # You can override vars by using host or group vars
 
+###########
+# GENERAL #
+###########
+
 # ACTIVATE BOTH FSID AND MONITOR_SECRET VARIABLES FOR NON-VAGRANT DEPLOYMENT
 fsid: "{{ cluster_uuid.stdout }}"
 # monitor_secret:
@@ -10,9 +14,10 @@ cephx: true
 # referenced in common role too.
 radosgw: false
 
-###########
-# OPENSTACK
-###########
+
+#############
+# OPENSTACK #
+#############
 
 openstack_config: false
 openstack_glance_pool: images
@@ -24,3 +29,13 @@ openstack_keys:
   - { name: client.glance, value: "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ openstack_glance_pool }}'" }
   - { name: client.cinder, value: "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ openstack_cinder_pool }}, allow rwx pool={{ openstack_nova_pool }}, allow rx pool={{ openstack_glance_pool }}'"  }
   - { name: client.cinder-backup, value: "mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ openstack_cinder_backup_pool }}'" }
+
+
+##########
+# DOCKER #
+##########
+
+ceph_containerized_deployment: false
+ceph_mon_docker_interface: eth0
+ceph_mon_docker_username: ceph
+ceph_mon_docker_imagename: "mon:latest"

roles/ceph-mon/tasks/docker.yml

@@ -0,0 +1,42 @@
+---
+- name: Try to fetch Ceph config and keys
+  copy: >
+    src=fetch/docker_mon_files/{{ item }}
+    dest=/etc/ceph/
+    owner=root
+    group=root
+    mode=600
+  with_items:
+    - /etc/ceph/ceph.client.admin.keyring
+    - /etc/ceph/ceph.conf
+    - /etc/ceph/monmap
+    - /etc/ceph/ceph.mon.keyring
+  ignore_errors: true
+
+- name: Run the Ceph Monitor docker image
+  docker: >
+    image="{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+    name=ceph-{{ ansible_hostname }}
+    net=host
+    state=running
+    env="MON_NAME=ceph-{{ ansible_hostname }},MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }}"
+    volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
+
+- name: Ensure ceph_mon service is running
+  docker: >
+    image="{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+    name="ceph-{{ ansible_hostname }}"
+    ports=6789:6789
+    detach=yes
+    state=running
+
+- name: Collect Ceph files to the Ansible server
+  fetch: >
+    src={{ item }}
+    dest=fetch/docker_mon_files/{{ item }}
+    flat=yes
+  with_items:
+    - /etc/ceph/ceph.client.admin.keyring
+    - /etc/ceph/ceph.conf
+    - /etc/ceph/monmap
+    - /etc/ceph/ceph.mon.keyring

roles/ceph-mon/tasks/main.yml

@@ -1,3 +1,12 @@
 ---
 - include: deploy_monitors.yml
+  when: not ceph_containerized_deployment
+
 - include: ceph_keys.yml
+  when: not ceph_containerized_deployment
+
+- include: ceph_keys.yml
+  when: not ceph_containerized_deployment
+
+- include: docker.yml
+  when: ceph_containerized_deployment

roles/ceph-osd/defaults/main.yml

@@ -104,3 +104,12 @@ osd_directories:
   - /var/lib/ceph/osd/mydir2
   - /var/lib/ceph/osd/mydir3
   - /var/lib/ceph/osd/mydir4
+
+
+##########
+# DOCKER #
+##########
+
+ceph_containerized_deployment: false
+ceph_osd_docker_username: ceph
+ceph_osd_docker_imagename: "osd:latest"

roles/ceph-osd/tasks/docker.yml

@@ -0,0 +1,29 @@
+---
+- name: Fetch Ceph config and keys
+  copy: >
+    src=fetch/docker_mon_files/{{ item }}
+    dest=/etc/ceph/
+    owner=root
+    group=root
+    mode=600
+  with_items:
+    - /etc/ceph/ceph.client.admin.keyring
+    - /etc/ceph/ceph.conf
+    - /etc/ceph/monmap
+    - /etc/ceph/ceph.mon.keyring
+
+- name: Run the Ceph OSD docker image
+  docker: >
+    image="{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
+    name=ceph-{{ ansible_hostname }}
+    net=host
+    state=running
+    env="OSD_NAME=ceph-{{ ansible_hostname }}"
+    volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
+
+- name: Ensure ceph_osd service is running
+  docker: >
+    image="{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
+    name="ceph-{{ ansible_hostname }}"
+    detach=yes
+    state=running

roles/ceph-osd/tasks/main.yml

@@ -1,32 +1,16 @@
 ---
-## Deploy Ceph Oject Storage Daemon(s)
 
-- name: Install dependencies
-  apt: >
-    pkg=parted
-    state=present
-  when: ansible_os_family == 'Debian'
-
-- name: Install dependencies
-  yum: >
-    name=parted
-    state=present
-  when: ansible_os_family == 'RedHat'
-
-- name: Copy OSD bootstrap key
-  copy: >
-    src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-osd/ceph.keyring
-    dest=/var/lib/ceph/bootstrap-osd/ceph.keyring
-    owner=root
-    group=root
-    mode=600
-  when: cephx
+- include: pre_requisite.yml
+  when: not ceph_containerized_deployment
 
 - include: journal_collocation.yml
-  when: journal_collocation
+  when: journal_collocation and not ceph_containerized_deployment
 
 - include: raw_multi_journal.yml
-  when: raw_multi_journal
+  when: raw_multi_journal and not ceph_containerized_deployment
 
 - include: osd_directory.yml
-  when: osd_directory
+  when: osd_directory and not ceph_containerized_deployment
+
+- include: docker.yml
+  when: ceph_containerized_deployment

roles/ceph-osd/tasks/pre_requisite.yml

@@ -0,0 +1,23 @@
+---
+## Deploy Ceph Oject Storage Daemon(s)
+
+- name: Install dependencies
+  apt: >
+    pkg=parted
+    state=present
+  when: ansible_os_family == 'Debian'
+
+- name: Install dependencies
+  yum: >
+    name=parted
+    state=present
+  when: ansible_os_family == 'RedHat'
+
+- name: Copy OSD bootstrap key
+  copy: >
+    src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-osd/ceph.keyring
+    dest=/var/lib/ceph/bootstrap-osd/ceph.keyring
+    owner=root
+    group=root
+    mode=600
+  when: cephx

roles/ceph-radosgw/defaults/main.yml

@@ -12,5 +12,14 @@ http_100_continue: false
 # Rados Gateway options
 redhat_distro_ceph_extra: centos6.4 # supported distros are centos6.3, centos6.4, centos6, fedora18, fedora19, opensuse12.2, rhel6.3, rhel6.4, rhel6.5, rhel6, sles11sp2
 email_address: foo@bar.com
-
 radosgw_frontend: civetweb # we currently only support a single backend
+
+
+##########
+# DOCKER #
+##########
+
+ceph_containerized_deployment: false
+ceph_rgw_civetweb_port: 80
+ceph_rgw_docker_username: ceph
+ceph_rgw_docker_imagename: "rgw:latest"

roles/ceph-radosgw/tasks/docker.yml

@@ -0,0 +1,29 @@
+---
+- name: Fetch Ceph config and keys
+  copy: >
+    src=fetch/docker_mon_files/{{ item }}
+    dest=/etc/ceph/
+    owner=root
+    group=root
+    mode=600
+  with_items:
+    - /etc/ceph/ceph.client.admin.keyring
+    - /etc/ceph/ceph.conf
+    - /etc/ceph/monmap
+    - /etc/ceph/ceph.mon.keyring
+
+- name: Run the Ceph Monitor docker image
+  docker: >
+    image="{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"
+    name=ceph-rgw-{{ ansible_hostname }}
+    net=host
+    state=running
+    env="RGW_NAME=ceph-rgw-{{ ansible_hostname }}, RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}"
+    volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
+
+- name: Ensure ceph_rgw service is running
+  docker: >
+    image="{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"
+    name="ceph-{{ ansible_hostname }}"
+    detach=yes
+    state=running

roles/ceph-radosgw/tasks/main.yml

@@ -1,31 +1,7 @@
 ---
-- name: Copy RGW bootstrap key
-  copy: >
-    src=fetch/{{ fsid }}/etc/ceph/radosgw.{{ ansible_hostname }}.keyring
-    dest=/etc/ceph/radosgw.{{ ansible_hostname }}.keyring
-    owner=root
-    group=root
-    mode=600
-  when: cephx
 
-- name: Set RGW bootstrap key permissions
-  file: >
-    path=/etc/ceph/radosgw.{{ ansible_hostname }}.keyring
-    mode=0600
-    owner=root
-    group=root
-  when: cephx
-
-- name: Create RGW directory
-  file: >
-    path=/var/lib/ceph/radosgw/{{ ansible_fqdn }}
-    state=directory
-    owner=root
-    group=root
-    mode=0644
-
-## Check OS family
-#
+- include: pre_requisite.yml
+  when: not ceph_containerized_deployment
 
 - include: install_redhat.yml
   when: ansible_os_family == 'RedHat' and radosgw_frontend == 'apache'
@@ -38,20 +14,15 @@
     pkg=radosgw
     state=present
     update_cache=yes
-  when: ansible_os_family == 'Debian'
+  when: ansible_os_family == 'Debian' and not ceph_containerized_deployment
 
 - name: "Install Rados Gateway"
   yum: >
     name=ceph-radosgw
     state=present
-  when: ansible_os_family == 'RedHat'
+  when: ansible_os_family == 'RedHat' and not ceph_containerized_deployment
 
-## If we don't perform this check Ansible will start multiple instance of radosgw
-- name: Check if RGW is started
-  command: /etc/init.d/radosgw status
-  register: rgwstatus
-  ignore_errors: True
+- include: start_radosgw.yml
 
-- name: Start RGW
-  command: /etc/init.d/radosgw start
-  when: rgwstatus.rc != 0
+- include: docker.yml
+  when: ceph_containerized_deployment

roles/ceph-radosgw/tasks/pre_requisite.yml

@@ -0,0 +1,25 @@
+---
+- name: Copy RGW bootstrap key
+  copy: >
+    src=fetch/{{ fsid }}/etc/ceph/radosgw.{{ ansible_hostname }}.keyring
+    dest=/etc/ceph/radosgw.{{ ansible_hostname }}.keyring
+    owner=root
+    group=root
+    mode=600
+  when: cephx
+
+- name: Set RGW bootstrap key permissions
+  file: >
+    path=/etc/ceph/radosgw.{{ ansible_hostname }}.keyring
+    mode=0600
+    owner=root
+    group=root
+  when: cephx
+
+- name: Create RGW directory
+  file: >
+    path=/var/lib/ceph/radosgw/{{ ansible_fqdn }}
+    state=directory
+    owner=root
+    group=root
+    mode=0644

roles/ceph-radosgw/tasks/start_radosgw.yml

@@ -0,0 +1,10 @@
+---
+## If we don't perform this check Ansible will start multiple instance of radosgw
+- name: Check if RGW is started
+  command: /etc/init.d/radosgw status
+  register: rgwstatus
+  ignore_errors: True
+
+- name: Start RGW
+  command: /etc/init.d/radosgw start
+  when: rgwstatus.rc != 0

site.yml

@@ -8,7 +8,7 @@
   - rgws
   sudo: True
   roles:
-  - ceph-common
+  - { role: ceph-common, when: not docker }
 
 - hosts: mons
   sudo: True