Adding dockerized ceph-mon deployment with kv backend

group_vars/all.docker.sample

@@ -5,6 +5,10 @@ dummy:
 # MON #
 #######
 #mon_containerized_deployment: true
+#mon_containerized_deployment_with_kv: false
+#kv_type: etcd
+#kv_endpoint: 127.0.0.1
+#kv_port: 4001
 #mon_docker_privileged: true
 #ceph_mon_docker_username: ceph
 #ceph_mon_docker_imagename: daemon

roles/ceph-common/templates/ceph.conf.j2

@@ -15,7 +15,6 @@ auth service required = none
 auth client required = none
 auth supported = none
 {% endif %}
-fsid = {{ fsid }}
 max open files = {{ max_open_files }}
 osd pool default pg num = {{ pool_default_pg_num }}
 osd pool default pgp num = {{ pool_default_pgp_num }}
@@ -88,6 +87,7 @@ debug mon = {{ debug_mon_level }}
 debug paxos = {{ debug_mon_level }}
 debug auth = {{ debug_mon_level }}
 {% endif %}
+{% if not mon_containerized_deployment_with_kv %}
 {% for host in groups[mon_group_name] %}
 {% if hostvars[host]['ansible_fqdn'] is defined and mon_use_fqdn %}
 [mon.{{ hostvars[host]['ansible_fqdn'] }}]
@@ -102,6 +102,7 @@ host = {{ hostvars[host]['ansible_hostname'] }}
 {% include 'mon_addr_address.j2' %}
 {% endif %}
 {% endfor %}
+{% endif %}
 
 [osd]
 osd mkfs type = {{ osd_mkfs_type }}

roles/ceph-mon/defaults/main.yml

@@ -63,8 +63,12 @@ openstack_keys:
 ##########
 
 mon_containerized_deployment: false
+mon_containerized_deployment_with_kv: false
+mon_containerized_default_ceph_conf_with_kv: false
 ceph_mon_docker_interface: eth0
 #ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface
 ceph_mon_docker_username: ceph
 ceph_mon_docker_imagename: daemon
 ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables
+ceph_docker_on_openstack: false
+mon_docker_privileged: true

roles/ceph-mon/tasks/docker/main.yml

@@ -14,15 +14,21 @@
       is_atomic='{{ stat_ostree.stat.exists }}'
                                   
 - include: checks.yml
-  when: ceph_health.rc != 0
+  when: ceph_health.rc != 0 and not mon_containerized_deployment_with_kv
 
 - include: pre_requisite.yml
+
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'
 
 # let the first mon create configs and keyrings
 - include: create_configs.yml
   when: inventory_hostname == groups.mons[0]
+
 - include: fetch_configs.yml
+  when: not mon_containerized_deployment_with_kv
+
 - include: start_docker_monitor.yml
-- include: ../ceph_keys.yml
+
+- include: ../ceph_keys.yml
+  when: not mon_containerized_deployment_with_kv

roles/ceph-mon/tasks/docker/start_docker_monitor.yml

@@ -6,11 +6,33 @@
           - /var/lib/ceph/bootstrap-osd/ceph.keyring
           - /var/lib/ceph/bootstrap-rgw/ceph.keyring
           - /var/lib/ceph/bootstrap-mds/ceph.keyring
-                                                
+  when: not mon_containerized_deployment_with_kv
+
 - name: pull ceph daemon image
   shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
 
-# Use systemd to manage container on Atomic host
+- name: populate kv_store
+  docker:
+    name: populate-kv-store
+    image: ceph/daemon
+    command: populate_kvstore
+    net: host
+    env:
+      KV_TYPE: "{{kv_type}}"
+      KV_IP: "{{kv_endpoint}}"
+      KV_PORT: "{{kv_port}}"
+    volumes:
+      - /etc/ceph/ceph.conf:/etc/ceph/ceph.defaults
+  run_once: true
+  when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv
+
+- name: delete populate-kv-store docker
+  docker:
+    name: populate-kv-store
+    state: absent
+    image: ceph/daemon
+
+# Use systemd to manage container on Atomic host and CoreOS
 - name: generate systemd unit file
   sudo: true
   template:
@@ -19,23 +41,23 @@
     owner: "root"
     group: "root"
     mode: "0644"
-  
+
 - name: link systemd unit file for mon instance
   file:
     src: /var/lib/ceph/ceph-mon@.service
     dest: /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
     state: link
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: enable systemd unit file for mon instance
   shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
   failed_when: false
   changed_when: false
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: systemd start mon container
   service:
@@ -43,7 +65,7 @@
     state: started
     enabled: yes
   changed_when: false
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: wait for ceph.conf exists
   wait_for:
@@ -59,7 +81,7 @@
     privileged: "{{ mon_docker_privileged }}"
     env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}"
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
-  when: not is_atomic 
+  when: not is_atomic and ansible_os_family != 'CoreOS'
 
 - name: stat for ceph config and keys
   stat: path="{{ item }}"
@@ -67,6 +89,7 @@
   changed_when: false
   failed_when: false
   register: statmonconfig
+  when: not mon_containerized_deployment_with_kv
 
 - name: fetch boostrap keys and conf from mon
   fetch:
@@ -76,5 +99,5 @@
   with_together:
     - ceph_bootstrap_config_keys
     - statmonconfig.results
-  when: item.1.stat.exists == true
+  when: not mon_containerized_deployment_with_kv and item.1.stat.exists == true
         and inventory_hostname == groups.mons[0]

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -3,12 +3,17 @@ Description=Ceph Monitor
 After=docker.service
 
 [Service]
-EnvironmentFile=/etc/environment
+EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker rm %i
 ExecStartPre=/usr/bin/mkdir -p /etc/ceph /var/lib/ceph/mon
-ExecStart=/usr/bin/docker run --rm --name %p --net=host \
+ExecStart=/usr/bin/docker run --rm --name %i --net=host \
+   {% if not mon_containerized_deployment_with_kv -%}
    -v /var/lib/ceph:/var/lib/ceph \
    -v /etc/ceph:/etc/ceph \
+   {% else -%}
+   -e KV_TYPE={{kv_type}} \
+   -e KV_IP={{kv_endpoint}}\
+   {% endif -%}
    --privileged \
    -e CEPH_DAEMON=MON \
    -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }} \