Merge pull request #661 from PiotrProkop/ceph-osd-kv

Adding ceph-osd continerized deployment with kv store.

group_vars/all.docker.sample

@@ -21,6 +21,9 @@ dummy:
 # OSD #
 #######
 #osd_containerized_deployment: true
+#osd_containerized_deployment_with_kv: false
+#kv_type: etcd
+#kv_endpoint: 127.0.0.1
 #ceph_osd_docker_username: ceph
 #ceph_osd_docker_imagename: daemon
 #ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE" # comma separated variables

group_vars/osds.sample

@@ -128,6 +128,10 @@ dummy:
 ##########
 
 #osd_containerized_deployment: false
+#osd_containerized_deployment_with_kv: false
+#kv_type: etcd
+#kv_endpoint: 127.0.0.1
+#ceph_osd_docker_prepare_env: ""
 #ceph_osd_docker_username: ceph
 #ceph_osd_docker_imagename: daemon
 #ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables

roles/ceph-osd/defaults/main.yml

@@ -120,6 +120,10 @@ bluestore: false
 ##########
 
 osd_containerized_deployment: false
+osd_containerized_deployment_with_kv: false
+kv_type: etcd
+kv_endpoint: 127.0.0.1
+ceph_osd_docker_prepare_env: ""
 ceph_osd_docker_username: ceph
 ceph_osd_docker_imagename: daemon
 ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables

roles/ceph-osd/tasks/docker/main.yml

@@ -6,7 +6,7 @@
   failed_when: false
 
 - include: checks.yml
-  when: ceph_health.rc != 0
+  when: ceph_health.rc != 0 and not osd_containerized_deployment_with_kv
 
 - name: check if it is Atomic host
   stat: path=/run/ostree-booted
@@ -17,8 +17,11 @@
       is_atomic='{{ stat_ostree.stat.exists }}'
 
 - include: pre_requisite.yml
+
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'
 
 - include: fetch_configs.yml
+  when: not osd_containerized_deployment_with_kv
+
 - include: start_docker_osd.yml

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -24,7 +24,21 @@
     env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_prepare_env }}"
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev/:/dev/"
   with_items: ceph_osd_docker_devices
-  when: ceph_osd_docker_prepare_env is defined
+  when: ceph_osd_docker_prepare_env is defined and not osd_containerized_deployment_with_kv
+
+- name: prepare ceph osd disk with kv_store
+  docker:
+    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
+    name: "{{ ansible_hostname }}-osd-prepare-{{ item | regex_replace('/', '') }}"
+    net: host
+    pid: host
+    state: running
+    privileged: yes
+    volumes: "/dev/:/dev/"
+    env: "OSD_DEVICE={{ item }},KV_TYPE={{kv_type}},KV_IP={{kv_endpoint}},{{ ceph_osd_docker_prepare_env }}"
+  ignore_errors: true
+  with_items: ceph_osd_docker_devices
+  when: ceph_osd_docker_prepare_env is defined and osd_containerized_deployment_with_kv
 
 # Use systemd to manage container on Atomic host
 - name: generate systemd unit file
@@ -43,18 +57,18 @@
     dest: /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service
     state: link
   with_items: ceph_osd_docker_devices
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: enable systemd unit file for osd instance
   shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service
   failed_when: false
   changed_when: false
   with_items: ceph_osd_docker_devices
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: systemd start osd container
   service:
@@ -63,7 +77,7 @@
     enabled: yes
   changed_when: false
   with_items: ceph_osd_docker_devices
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: run the ceph osd docker image
   docker:
@@ -76,4 +90,4 @@
     env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_extra_env }}"
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev/:/dev/"
   with_items: ceph_osd_docker_devices
-  when: not is_atomic
+  when: not is_atomic and ansible_os_family != 'CoreOS'

roles/ceph-osd/templates/ceph-osd.service.j2

@@ -3,12 +3,17 @@ Description=Ceph OSD
 After=docker.service
 
 [Service]
-EnvironmentFile=/etc/environment
+EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
 ExecStartPre=-/usr/bin/docker rm {{ ansible_hostname }}-osd-dev%i
 ExecStart=/usr/bin/docker run --rm --net=host --pid=host\
+   {% if not osd_containerized_deployment_with_kv -%}
    -v /var/lib/ceph:/var/lib/ceph \
    -v /etc/ceph:/etc/ceph \
+   {% else -%}
+   -e KV_TYPE={{kv_type}} \
+   -e KV_IP={{kv_endpoint}}\
+   {% endif -%}
    -v /dev:/dev \
    --privileged \
    -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \