Merge pull request #286 from ceph/deploy-on-docker

Update the Docker bootstrap part

group_vars/mdss

@@ -0,0 +1,13 @@
+---
+dummy:
+
+#cephx: true
+
+
+##########
+# DOCKER #
+##########
+
+#ceph_containerized_deployment: false
+#ceph_mds_docker_username: ceph
+#ceph_mds_docker_imagename: daemon

group_vars/mons

@@ -62,6 +62,7 @@ dummy:
 ##########
 
 #ceph_containerized_deployment: false
-#ceph_mon_docker_interface: eth0
 #ceph_mon_docker_username: ceph
-#ceph_mon_docker_imagename: "mon:latest"
+#ceph_mon_docker_imagename: "daemon"
+#ceph_mon_docker_interface: eth0
+#ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface

group_vars/osds

@@ -105,3 +105,14 @@ osd_crush_location: "'root={{ ceph_crush_root }} rack={{ ceph_crush_rack }} host
 #  - /var/lib/ceph/osd/mydir2
 #  - /var/lib/ceph/osd/mydir3
 #  - /var/lib/ceph/osd/mydir4
+
+
+##########
+# DOCKER #
+##########
+
+#ceph_containerized_deployment: false
+#ceph_osd_docker_username: ceph
+#ceph_osd_docker_imagename: daemon
+#ceph_osd_docker_devices:
+#  - /dev/sdb

group_vars/rgws

@@ -0,0 +1,25 @@
+---
+dummy:
+
+## Ceph options
+#
+#cephx: true
+
+# Toggle 100-continue support for Apache and FastCGI
+# WARNING: Changing this value will cause an outage of Apache while it is reinstalled on RGW nodes
+#http_100_continue: false
+
+# Rados Gateway options
+#redhat_distro_ceph_extra: centos6.4 # supported distros are centos6.3, centos6.4, centos6, fedora18, fedora19, opensuse12.2, rhel6.3, rhel6.4, rhel6.5, rhel6, sles11sp2
+#email_address: foo@bar.com
+#radosgw_frontend: civetweb # we currently only support a single backend
+
+
+##########
+# DOCKER #
+##########
+
+#ceph_containerized_deployment: false
+#ceph_rgw_docker_username: ceph
+#ceph_rgw_docker_imagename: daemon
+#ceph_rgw_civetweb_port: 80

roles/ceph-mds/defaults/main.yml

@@ -10,4 +10,4 @@ cephx: true
 
 ceph_containerized_deployment: false
 ceph_mds_docker_username: ceph
-ceph_mds_docker_imagename: "mds:latest"
+ceph_mds_docker_imagename: daemon

roles/ceph-mds/meta/main.yml

@@ -1,3 +1,3 @@
 ---
 dependencies:
-  - { role: ceph-common}
+  - { role: ceph-common, when: not docker }

roles/ceph-mds/tasks/docker.yml

@@ -1,29 +1,46 @@
 ---
-- name: fetch Ceph config and keys
-  copy: >
-    src=fetch/docker_mon_files/{{ item }}
-    dest=/etc/ceph/
-    owner=root
-    group=root
-    mode=600
-  with_items:
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
       - /etc/ceph/ceph.client.admin.keyring
       - /etc/ceph/ceph.conf
       - /etc/ceph/monmap
       - /etc/ceph/ceph.mon.keyring
+      - /var/lib/ceph/bootstrap-mds/ceph.keyring
 
-- name: run the Ceph Medata docker image
+- name: install docker-py
+  pip: name=docker-py version=1.1.0 # https://github.com/ansible/ansible-modules-core/issues/1227
+
+- name: stat for Ceph config and keys
+  stat: >
+    path={{ item }}
+  with_items: ceph_config_keys
+  register: statconfig
+  ignore_errors: True
+
+- name: try to fetch Ceph config and keys
+  copy: >
+    src=fetch/docker_mon_files/"{{ item }}"
+    dest=/etc/ceph/
+    owner=root
+    group=root
+    mode=600
+  with_together:
+    - ceph_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == False
+
+- name: run the ceph medata docker image
   docker: >
     image="{{ ceph_mds_docker_username }}/{{ ceph_mds_docker_imagename }}"
-    name=ceph-mds-{{ ansible_hostname }}
+    name=ceph-{{ ansible_hostname }}-mds
     net=host
     state=running
-    env="MDS_NAME=ceph-mds-{{ ansible_hostname }}, MDS_CIVETWEB_PORT={{ ceph_mds_civetweb_port }}"
+    env="CEPH_DAEMON=MDS,CEPHFS_CREATE=1"
     volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
 
 - name: ensure ceph_mds service is running
   docker: >
     image="{{ ceph_mds_docker_username }}/{{ ceph_mds_docker_imagename }}"
-    name="ceph-{{ ansible_hostname }}"
-    detach=yes
+    name="ceph-{{ ansible_hostname }}-mds"
     state=running

roles/ceph-mon/defaults/main.yml

@@ -59,5 +59,6 @@ openstack_keys:
 
 ceph_containerized_deployment: false
 ceph_mon_docker_interface: eth0
+#ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface
 ceph_mon_docker_username: ceph
-ceph_mon_docker_imagename: "mon:latest"
+ceph_mon_docker_imagename: daemon

roles/ceph-mon/meta/main.yml

@@ -1,3 +1,3 @@
 ---
 dependencies:
-  - { role: ceph-common}
+  - { role: ceph-common, when: not docker }

roles/ceph-mon/tasks/create_mds_filesystems.yml

@@ -1,11 +1,17 @@
 ---
+# NOTE (leseb): in the present playbook the conditional is done on the task
+# We don't do this in main.yml because of the 'docker' variable, when set to true
+# the role 'ceph-common' doesn't get inherited so the condition can not be evaluate
+# since those check are performed by the ceph-common role
 - name: create filesystem pools
   command: ceph osd pool create {{ item }} {{ pool_default_pg_num }}
   with_items:
     - cephfs_data
     - cephfs_metadata
   changed_when: False
+  when: not {{ ceph_version.stdout | version_compare('0.84', '<') }}
 
 - name: create Ceph Filesystem
   command: ceph fs new {{ cephfs }} {{ cephfs_metadata }} {{ cephfs_data }}
   changed_when: False
+  when: not {{ ceph_version.stdout | version_compare('0.84', '<') }}

roles/ceph-mon/tasks/docker.yml

@@ -1,17 +1,33 @@
 ---
-- name: try to fetch Ceph config and keys
-  copy: >
-    src=fetch/docker_mon_files/{{ item }}
-    dest=/etc/ceph/
-    owner=root
-    group=root
-    mode=600
-  with_items:
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
       - /etc/ceph/ceph.client.admin.keyring
       - /etc/ceph/ceph.conf
       - /etc/ceph/monmap
       - /etc/ceph/ceph.mon.keyring
-  ignore_errors: true
+
+- name: install docker-py
+  pip: name=docker-py version=1.1.0 # https://github.com/ansible/ansible-modules-core/issues/1227
+
+- name: stat for Ceph config and keys
+  stat: >
+    path={{ item }}
+  with_items: ceph_config_keys
+  register: statconfig
+  ignore_errors: True
+
+- name: try to fetch Ceph config and keys
+  copy: >
+    src=fetch/docker_mon_files/"{{ item }}"
+    dest=/etc/ceph/
+    owner=root
+    group=root
+    mode=600
+  with_together:
+    - ceph_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == False
 
 - name: run the Ceph Monitor docker image
   docker: >
@@ -19,7 +35,7 @@
     name=ceph-{{ ansible_hostname }}
     net=host
     state=running
-    env="MON_NAME=ceph-{{ ansible_hostname }},MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }}"
+    env="MON_NAME=ceph-{{ ansible_hostname }},MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON, CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }}"
     volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
 
 - name: ensure ceph_mon service is running
@@ -40,3 +56,6 @@
     - /etc/ceph/ceph.conf
     - /etc/ceph/monmap
     - /etc/ceph/ceph.mon.keyring
+    - /var/lib/ceph/bootstrap-osd/ceph.keyring
+    - /var/lib/ceph/bootstrap-rgw/ceph.keyring
+    - /var/lib/ceph/bootstrap-mds/ceph.keyring

roles/ceph-mon/tasks/main.yml

@@ -9,10 +9,10 @@
   when: not ceph_containerized_deployment
 
 - include: create_mds_filesystems.yml
-  when: not ceph_containerized_deployment and not {{ ceph_version.stdout | version_compare('0.84', '<') }} and mds
+  when: not ceph_containerized_deployment and mds
 
 - include: secure_cluster.yml
-  when: secure_cluster and {{ ceph_version.stdout | version_compare('0.94', '>=') }}
+  when: secure_cluster and not ceph_containerized_deployment
   tags: secure_cluster
 
 - include: docker.yml

roles/ceph-mon/tasks/secure_cluster.yml

@@ -2,9 +2,11 @@
 - name: collect all the pool
   command: rados lspools
   register: ceph_pools
+  when: "{{ ceph_version.stdout | version_compare('0.94', '>=') }}"
 
 - name: secure the cluster
   command: ceph osd pool set {{ item[0] }} {{ item[1] }} true
   with_nested:
     - ceph_pools.stdout_lines
     - secure_cluster_flags
+  when: "{{ ceph_version.stdout | version_compare('0.94', '>=') }}"

roles/ceph-osd/defaults/main.yml

@@ -110,4 +110,5 @@ osd_directories:
 
 ceph_containerized_deployment: false
 ceph_osd_docker_username: ceph
-ceph_osd_docker_imagename: "osd:latest"
+ceph_osd_docker_imagename: daemon
+#ceph_osd_docker_device: /dev/sdb

roles/ceph-osd/meta/main.yml

@@ -1,3 +1,3 @@
 ---
 dependencies:
-  - { role: ceph-common}
+  - { role: ceph-common, when: not docker }

roles/ceph-osd/tasks/docker.yml

@@ -1,29 +1,49 @@
 ---
-- name: fetch Ceph config and keys
-  copy: >
-    src=fetch/docker_mon_files/{{ item }}
-    dest=/etc/ceph/
-    owner=root
-    group=root
-    mode=600
-  with_items:
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
       - /etc/ceph/ceph.client.admin.keyring
       - /etc/ceph/ceph.conf
       - /etc/ceph/monmap
       - /etc/ceph/ceph.mon.keyring
+      - /var/lib/ceph/bootstrap-osd/ceph.keyring
+
+- name: install docker-py
+  pip: name=docker-py version=1.1.0 # https://github.com/ansible/ansible-modules-core/issues/1227
+
+- name: stat for Ceph config and keys
+  stat: >
+    path={{ item }}
+  with_items: ceph_config_keys
+  register: statconfig
+  ignore_errors: True
+
+- name: try to fetch Ceph config and keys
+  copy: >
+    src=fetch/docker_mon_files/"{{ item }}"
+    dest=/etc/ceph/
+    owner=root
+    group=root
+    mode=600
+  with_together:
+    - ceph_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == False
 
 - name: run the Ceph OSD docker image
   docker: >
     image="{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
-    name=ceph-{{ ansible_hostname }}
+    name={{ ansible_hostname }}-osd-{{ item | regex_replace('/', '') }}
     net=host
     state=running
-    env="OSD_NAME=ceph-{{ ansible_hostname }}"
-    volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
+    privileged=yes
+    env="CEPH_DAEMON=OSD_CEPH_DISK,OSD_DEVICE={{ item }}"
+    volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev/:/dev/"
+  with_items: ceph_osd_docker_devices
 
 - name: ensure ceph_osd service is running
   docker: >
     image="{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
-    name="ceph-{{ ansible_hostname }}"
-    detach=yes
-    state=running
+    name={{ ansible_hostname }}-osd-{{ item | regex_replace('/', '') }}
+    state=started
+  with_items: ceph_osd_docker_devices

roles/ceph-radosgw/defaults/main.yml

@@ -22,4 +22,4 @@ radosgw_frontend: civetweb # we currently only support a single backend
 ceph_containerized_deployment: false
 ceph_rgw_civetweb_port: 80
 ceph_rgw_docker_username: ceph
-ceph_rgw_docker_imagename: "rgw:latest"
+ceph_rgw_docker_imagename: daemon

roles/ceph-radosgw/meta/main.yml

@@ -1,3 +1,3 @@
 ---
 dependencies:
-  - { role: ceph-common}
+  - { role: ceph-common, when: not docker }

roles/ceph-radosgw/tasks/docker.yml

@@ -1,29 +1,47 @@
 ---
-- name: fetch Ceph config and keys
-  copy: >
-    src=fetch/docker_mon_files/{{ item }}
-    dest=/etc/ceph/
-    owner=root
-    group=root
-    mode=600
-  with_items:
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
       - /etc/ceph/ceph.client.admin.keyring
       - /etc/ceph/ceph.conf
       - /etc/ceph/monmap
       - /etc/ceph/ceph.mon.keyring
+      - /var/lib/ceph/bootstrap-rgw/ceph.keyring
 
-- name: run the Ceph Monitor docker image
+- name: install docker-py
+  pip: name=docker-py version=1.1.0 # https://github.com/ansible/ansible-modules-core/issues/1227
+
+- name: stat for Ceph config and keys
+  stat: >
+    path={{ item }}
+  with_items: ceph_config_keys
+  register: statconfig
+  ignore_errors: True
+
+- name: try to fetch Ceph config and keys
+  copy: >
+    src=fetch/docker_mon_files/"{{ item }}"
+    dest=/etc/ceph/
+    owner=root
+    group=root
+    mode=600
+  with_together:
+    - ceph_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == False
+
+- name: run the rados gateway docker image
   docker: >
     image="{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"
-    name=ceph-rgw-{{ ansible_hostname }}
-    net=host
+    name=ceph-{{ ansible_hostname }}-rgw
+    expose={{ ceph_rgw_civetweb_port }}
     state=running
-    env="RGW_NAME=ceph-rgw-{{ ansible_hostname }}, RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}"
+    env="RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }},CEPH_DAEMON=RGW"
     volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
 
 - name: ensure ceph_rgw service is running
   docker: >
     image="{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"
-    name="ceph-{{ ansible_hostname }}"
+    name="ceph-{{ ansible_hostname }}-rgw"
     detach=yes
     state=running

roles/ceph-restapi/meta/main.yml

@@ -1,3 +1,3 @@
 ---
 dependencies:
-  - { role: ceph-common}
+  - { role: ceph-common, when: not docker }