Merge pull request #346 from ceph/fix-multi-mon-docker

Fix multi-mon docker deployment

group_vars/mons.sample

@@ -58,3 +58,4 @@ dummy:
 #ceph_mon_docker_imagename: "daemon"
 #ceph_mon_docker_interface: eth0
 #ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface
+#ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables

roles/ceph-mon/defaults/main.yml

@@ -56,3 +56,4 @@ ceph_mon_docker_interface: eth0
 #ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface
 ceph_mon_docker_username: ceph
 ceph_mon_docker_imagename: daemon
+ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables

roles/ceph-mon/tasks/docker.yml

@@ -6,59 +6,60 @@
       - /etc/ceph/ceph.conf
       - /etc/ceph/monmap
       - /etc/ceph/ceph.mon.keyring
+      - /var/lib/ceph/bootstrap-osd/ceph.keyring
+      - /var/lib/ceph/bootstrap-rgw/ceph.keyring
+      - /var/lib/ceph/bootstrap-mds/ceph.keyring
 
-# NOTE (leseb): force version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
+- name: create bootstrap directories
+  file: >
+    path=/var/lib/ceph/bootstrap-{{ item }}
+    state=directory
+  with_items:
+    - osd
+    - mds
+    - rgw
+
+# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
 - name: install docker-py
   pip: >
     name=docker-py
     version=1.1.0
 
 - name: stat for ceph config and keys
-  stat: >
-    path={{ item }}
+  local_action: stat path={{ item }}
   with_items: ceph_config_keys
+  changed_when: false
+  sudo: false
   ignore_errors: true
   register: statconfig
 
 - name: try to fetch ceph config and keys
   copy: >
-    src=fetch/docker_mon_files/"{{ item }}"
-    dest=/etc/ceph/
+    src=fetch/docker_mon_files/{{ item.0 }}
+    dest={{ item.0 }}
     owner=root
     group=root
-    mode=600
+    mode=644
   with_together:
     - ceph_config_keys
     - statconfig.results
   when: item.1.stat.exists == true
 
-- name: run the ceph Monitor docker image
+- name: run the ceph monitor docker image
   docker: >
     image="{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
-    name=ceph-{{ ansible_hostname }}
-    net=host
-    state=running
-    env="MON_NAME=ceph-{{ ansible_hostname }},MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON, CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }}"
+    name="{{ ansible_hostname }}"
+    net="host"
+    state="running"
+    env="MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}"
     volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
 
-- name: ensure ceph_mon service is running
-  docker: >
-    image="{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
-    name="ceph-{{ ansible_hostname }}"
-    ports=6789:6789
-    detach=yes
-    state=running
-
-- name: collect ceph files to the ansible server
+- name: push ceph files to the ansible server
   fetch: >
-    src={{ item }}
-    dest=fetch/docker_mon_files/{{ item }}
+    src={{ item.0 }}
+    dest=fetch/docker_mon_files/{{ item.0 }}
     flat=yes
-  with_items:
-    - /etc/ceph/ceph.client.admin.keyring
-    - /etc/ceph/ceph.conf
-    - /etc/ceph/monmap
-    - /etc/ceph/ceph.mon.keyring
-    - /var/lib/ceph/bootstrap-osd/ceph.keyring
-    - /var/lib/ceph/bootstrap-rgw/ceph.keyring
-    - /var/lib/ceph/bootstrap-mds/ceph.keyring
+  with_together:
+    - ceph_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == false

roles/ceph-mon/tasks/docker/copy_configs.yml

@@ -0,0 +1,10 @@
+---
+- name: push ceph files to the ansible server
+  fetch: >
+    src={{ item.0 }}
+    dest=fetch/docker_mon_files/{{ item.0 }}
+    flat=yes
+  with_together:
+    - ceph_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == false

roles/ceph-mon/tasks/docker/fetch_configs.yml

@@ -0,0 +1,31 @@
+---
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
+      - /etc/ceph/ceph.client.admin.keyring
+      - /etc/ceph/ceph.conf
+      - /etc/ceph/monmap
+      - /etc/ceph/ceph.mon.keyring
+      - /var/lib/ceph/bootstrap-osd/ceph.keyring
+      - /var/lib/ceph/bootstrap-rgw/ceph.keyring
+      - /var/lib/ceph/bootstrap-mds/ceph.keyring
+
+- name: stat for ceph config and keys
+  local_action: stat path={{ item }}
+  with_items: ceph_config_keys
+  changed_when: false
+  sudo: false
+  ignore_errors: true
+  register: statconfig
+
+- name: try to fetch ceph config and keys
+  copy: >
+    src=fetch/docker_mon_files/{{ item.0 }}
+    dest={{ item.0 }}
+    owner=root
+    group=root
+    mode=644
+  with_together:
+    - ceph_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == true

roles/ceph-mon/tasks/docker/main.yml

@@ -0,0 +1,5 @@
+---
+- include: pre_requisite.yml
+- include: fetch_configs.yml
+- include: start_docker_monitor.yml
+- include: copy_configs.yml

roles/ceph-mon/tasks/docker/pre_requisite.yml

@@ -0,0 +1,15 @@
+---
+- name: create bootstrap directories
+  file: >
+    path=/var/lib/ceph/bootstrap-{{ item }}
+    state=directory
+  with_items:
+    - osd
+    - mds
+    - rgw
+
+# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
+- name: install docker-py
+  pip: >
+    name=docker-py
+    version=1.1.0

roles/ceph-mon/tasks/docker/start_docker_monitor.yml

@@ -0,0 +1,9 @@
+---
+- name: run the ceph Monitor docker image
+  docker: >
+    image="{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+    name="{{ ansible_hostname }}"
+    net="host"
+    state="running"
+    env="MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}"
+    volumes="/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"

roles/ceph-mon/tasks/main.yml

@@ -18,5 +18,5 @@
     secure_cluster and
     not ceph_containerized_deployment
 
-- include: docker.yml
+- include: ./docker/main.yml
   when: ceph_containerized_deployment