ceph-ansible: allow to run containerized daemons

run containerized daemons in virtual machines.
to enable it simply do:

`cp site-docker.yml.sample site-docker.yml`

and set `docker: true` in `vagrant_variables.yml`

Signed-off-by: Sébastien Han <seb@redhat.com>

.gitignore

@@ -11,4 +11,5 @@ group_vars/rgws
 group_vars/restapis
 *.DS_Store
 site.yml
+site-docker.yml
 *.pyc

Vagrantfile

@@ -17,9 +17,15 @@ BOX        = settings['vagrant_box']
 MEMORY     = settings['memory']
 STORAGECTL = settings['vagrant_storagectl']
 ETH        = settings['eth']
+DOCKER     = settings['docker']
 
 ansible_provision = proc do |ansible|
-  ansible.playbook = 'site.yml'
+  if DOCKER then
+    ansible.playbook = 'site-docker.yml'
+  else
+    ansible.playbook = 'site.yml'
+  end
+
   # Note: Can't do ranges like mon[0-2] in groups because
   # these aren't supported by Vagrant, see
   # https://github.com/mitchellh/vagrant/issues/3539
@@ -33,16 +39,31 @@ ansible_provision = proc do |ansible|
   }
 
   # In a production deployment, these should be secret
-  ansible.extra_vars = {
-    ceph_stable: 'true',
-    journal_collocation: 'true',
-    journal_size: 100,
-    monitor_interface: ETH,
-    cluster_network: "#{SUBNET}.0/24",
-    public_network: "#{SUBNET}.0/24",
-    devices: settings['disks'],
-    os_tuning_params: settings['os_tuning_params']
-  }
+  if DOCKER then
+    ansible.extra_vars = {
+      mon_containerized_deployment: 'true',
+      osd_containerized_deployment: 'true',
+      mds_containerized_deployment: 'true',
+      rgw_containerized_deployment: 'true',
+      restapi_containerized_deployment: 'true',
+      ceph_mon_docker_interface: ETH,
+      ceph_mon_docker_subnet: "#{SUBNET}.0/24",
+      ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK,OSD_JOURNAL_SIZE=100",
+      ceph_osd_docker_device: settings['disks'],
+      ceph_rgw_civetweb_port: 8080
+    }
+  else
+    ansible.extra_vars = {
+      ceph_stable: 'true',
+      journal_collocation: 'true',
+      journal_size: 100,
+      monitor_interface: ETH,
+      cluster_network: "#{SUBNET}.0/24",
+      public_network: "#{SUBNET}.0/24",
+      devices: settings['disks'],
+      os_tuning_params: settings['os_tuning_params']
+    }
+  end
   ansible.limit = 'all'
 end
 

roles/ceph-mds/tasks/docker/fetch_configs.yml

@@ -9,7 +9,7 @@
       - /var/lib/ceph/bootstrap-mds/ceph.keyring
 
 - name: stat for ceph config and keys
-  local_action: stat path={{ item }}
+  local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
   with_items: ceph_config_keys
   changed_when: false
   become: false

roles/ceph-mds/tasks/docker/pre_requisite.yml

@@ -7,24 +7,45 @@
     - /etc/ceph/
     - /var/lib/ceph/bootstrap-mds
 
-- name: install pip on debian
+- name: install pip and docker on ubuntu
   apt:
-    name: python-pip
+    name: "{{ item }}"
     state: present
-  when: ansible_os_family == 'Debian'
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker
+    - docker.io
+  when: ansible_distribution == 'Ubuntu'
 
-- name: install pip on redhat
-  yum:
-    name: python-pip
+- name: install pip and docker on debian
+  apt:
+    name: "{{ item }}"
     state: present
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker-engine
+  when: ansible_distribution == 'Debian'
+
+- name: install pip and docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "yum"
 
-- name: install pip on redhat
+- name: install pip and docker on redhat
   dnf:
-    name: python-pip
+    name: "{{ item }}"
     state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"

roles/ceph-mon/tasks/docker/fetch_configs.yml

@@ -11,7 +11,7 @@
       - /var/lib/ceph/bootstrap-mds/ceph.keyring
 
 - name: stat for ceph config and keys
-  local_action: stat path={{ item }}
+  local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
   with_items: ceph_config_keys
   changed_when: false
   become: false

roles/ceph-mon/tasks/docker/pre_requisite.yml

@@ -9,24 +9,45 @@
     - /var/lib/ceph/bootstrap-mds
     - /var/lib/ceph/bootstrap-rgw
 
-- name: install pip on debian
+- name: install pip and docker on ubuntu
   apt:
-    name: python-pip
+    name: "{{ item }}"
     state: present
-  when: ansible_os_family == 'Debian'
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker
+    - docker.io
+  when: ansible_distribution == 'Ubuntu'
 
-- name: install pip on redhat
-  yum:
-    name: python-pip
+- name: install pip and docker on debian
+  apt:
+    name: "{{ item }}"
     state: present
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker-engine
+  when: ansible_distribution == 'Debian'
+
+- name: install pip and docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "yum"
 
-- name: install pip on redhat
+- name: install pip and docker on redhat
   dnf:
-    name: python-pip
+    name: "{{ item }}"
     state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"

roles/ceph-osd/tasks/docker/fetch_configs.yml

@@ -6,7 +6,7 @@
       - /var/lib/ceph/bootstrap-osd/ceph.keyring
 
 - name: stat for ceph config and keys
-  local_action: stat path={{ item }}
+  local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
   with_items: ceph_config_keys
   changed_when: false
   become: false

roles/ceph-osd/tasks/docker/pre_requisite.yml

@@ -7,24 +7,45 @@
     - /etc/ceph/
     - /var/lib/ceph/bootstrap-osd
 
-- name: install pip on debian
+- name: install pip and docker on ubuntu
   apt:
-    name: python-pip
+    name: "{{ item }}"
     state: present
-  when: ansible_os_family == 'Debian'
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker
+    - docker.io
+  when: ansible_distribution == 'Ubuntu'
 
-- name: install pip on redhat
-  yum:
-    name: python-pip
+- name: install pip and docker on debian
+  apt:
+    name: "{{ item }}"
     state: present
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker-engine
+  when: ansible_distribution == 'Debian'
+
+- name: install pip and docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "yum"
 
-- name: install pip on redhat
+- name: install pip and docker on redhat
   dnf:
-    name: python-pip
+    name: "{{ item }}"
     state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"

roles/ceph-restapi/tasks/docker/fetch_configs.yml

@@ -6,7 +6,7 @@
       - /etc/ceph/ceph.client.admin.keyring
 
 - name: stat for ceph config and keys
-  local_action: stat path={{ item }}
+  local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
   with_items: ceph_config_keys
   changed_when: false
   become: false

roles/ceph-restapi/tasks/docker/pre_requisite.yml

@@ -1,22 +1,43 @@
 ---
-- name: install pip on debian
+- name: install pip and docker on ubuntu
   apt:
-    name: python-pip
+    name: "{{ item }}"
     state: present
-  when: ansible_os_family == 'Debian'
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker
+    - docker.io
+  when: ansible_distribution == 'Ubuntu'
 
-- name: install pip on redhat
-  yum:
-    name: python-pip
+- name: install pip and docker on debian
+  apt:
+    name: "{{ item }}"
     state: present
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker-engine
+  when: ansible_distribution == 'Debian'
+
+- name: install pip and docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "yum"
 
-- name: install pip on redhat
+- name: install pip and docker on redhat
   dnf:
-    name: python-pip
+    name: "{{ item }}"
     state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"

roles/ceph-rgw/tasks/docker/fetch_configs.yml

@@ -6,7 +6,7 @@
       - /var/lib/ceph/bootstrap-rgw/ceph.keyring
 
 - name: stat for ceph config and keys
-  local_action: stat path={{ item }}
+  local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
   with_items: ceph_config_keys
   changed_when: false
   become: false

roles/ceph-rgw/tasks/docker/pre_requisite.yml

@@ -7,24 +7,45 @@
     - /etc/ceph/
     - /var/lib/ceph/bootstrap-rgw
 
-- name: install pip on debian
+ name: install pip and docker on ubuntu
   apt:
-    name: python-pip
+    name: "{{ item }}"
     state: present
-  when: ansible_os_family == 'Debian'
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker
+    - docker.io
+  when: ansible_distribution == 'Ubuntu'
 
-- name: install pip on redhat
-  yum:
-    name: python-pip
+- name: install pip and docker on debian
+  apt:
+    name: "{{ item }}"
     state: present
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker-engine
+  when: ansible_distribution == 'Debian'
+
+- name: install pip and docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "yum"
 
-- name: install pip on redhat
+- name: install pip and docker on redhat
   dnf:
-    name: python-pip
+    name: "{{ item }}"
     state: present
+  with_items:
+    - python-pip
+    - docker-engine
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"

site-docker.yml.sample

@@ -0,0 +1,28 @@
+---
+# Defines deployment design and assigns role to server groups
+
+- hosts: mons
+  become: True
+  roles:
+  - ceph-mon
+  serial: 1 # MUST be '1' WHEN DEPLOYING MONITORS ON DOCKER CONTAINERS
+
+- hosts: osds
+  become: True
+  roles:
+  - ceph-osd
+
+- hosts: mdss
+  become: True
+  roles:
+  - ceph-mds
+
+- hosts: rgws
+  become: True
+  roles:
+  - ceph-rgw
+
+- hosts: restapis
+  become: True
+  roles:
+  - ceph-restapi

site.yml.sample

@@ -5,7 +5,6 @@
   become: True
   roles:
   - ceph-mon
-  #serial: 1 # ENABLE THIS WHEN DEPLOYING MONITORS ON DOCKER CONTAINERS
 
 - hosts: osds
   become: True

vagrant_variables.yml.sample

@@ -1,5 +1,8 @@
 ---
 
+# DEPLOY CONTAINERIZED DAEMONS
+docker: false
+
 # DEFINE THE NUMBER OF VMS TO RUN
 mon_vms: 3
 osd_vms: 3