Merge pull request #865 from PiotrProkop/containerized-osd-with-kv-fix

Fixing containerized deployment with kv

roles/ceph-osd/defaults/main.yml

@@ -123,6 +123,7 @@ osd_containerized_deployment: false
 osd_containerized_deployment_with_kv: false
 kv_type: etcd
 kv_endpoint: 127.0.0.1
+kv_port: 4001
 ceph_osd_docker_prepare_env: "OSD_FORCE_ZAP=1"
 ceph_osd_docker_username: ceph
 ceph_osd_docker_imagename: daemon

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -41,16 +41,20 @@
     - not osd_containerized_deployment_with_kv
 
 - name: prepare ceph osd disk with kv_store
-  docker:
-    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}"
-    name: "{{ ansible_hostname }}-osd-prepare-{{ item.0 | regex_replace('/', '') }}"
-    net: host
-    pid: host
-    state: running
-    privileged: yes
-    volumes: "{{ item.0 }}:{{ item.0 }}"
-    env: "OSD_DEVICE={{ item.0 }},KV_TYPE={{kv_type}},KV_IP={{kv_endpoint}},{{ ceph_osd_docker_prepare_env }}"
-  ignore_errors: true
+  shell: |
+    docker run  --net=host   \
+    --pid=host \
+    --privileged=true \
+    --name="{{ ansible_hostname }}-osd-prepare-{{ item.0 |
+    regex_replace('/', '') }}" \
+    -v /dev:/dev \
+    -e "OSD_DEVICE={{ item.0 }}" \
+    -e "{{ ceph_osd_docker_prepare_env }}" \
+    -e CEPH_DAEMON=osd_ceph_disk_prepare \
+    -e KV_TYPE={{kv_type}} \
+    -e KV_IP={{kv_endpoint}} \
+    -e KV_PORT={{kv_port}} \
+    "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}" \
   with_together:
     - ceph_osd_docker_devices
     - osd_prepared.results
@@ -64,23 +68,15 @@
   become: true
   template:
     src: "{{ role_path }}/templates/ceph-osd.service.j2"
-    dest: /var/lib/ceph/ceph-osd@.service
+    dest: /etc/systemd/system/ceph-osd@.service
     owner: "root"
     group: "root"
     mode: "0644"
   failed_when: false
   when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
-- name: link systemd unit file for osd instance
-  file:
-    src: /var/lib/ceph/ceph-osd@.service
-    dest: /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service
-    state: link
-  with_items: ceph_osd_docker_devices
-  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
-
 - name: enable systemd unit file for osd instance
-  shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service
+  shell: systemctl enable ceph-osd@{{ item | basename }}.service
   failed_when: false
   changed_when: false
   with_items: ceph_osd_docker_devices
@@ -103,7 +99,7 @@
 
 - name: run the ceph osd docker image
   docker:
-    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
+    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}"
     name: "{{ ansible_hostname }}-osd-{{ item | regex_replace('/', '') }}"
     net: host
     pid: host
@@ -119,7 +115,7 @@
 
 - name: run the ceph osd docker image with kv
   docker:
-    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
+    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}"
     name: "{{ ansible_hostname }}-osd-{{ item | regex_replace('/', '') }}"
     net: host
     pid: host
@@ -132,3 +128,4 @@
    - ansible_os_family != 'RedHat'
    - ansible_os_family != 'CoreOS'
    - osd_containerized_deployment_with_kv
+ 

roles/ceph-osd/templates/ceph-osd.service.j2

@@ -5,7 +5,7 @@ After=docker.service
 [Service]
 EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
-ExecStartPre=-/usr/bin/docker rm {{ ansible_hostname }}-osd-dev%i
+ExecStartPre=-/usr/bin/docker rm -f  {{ ansible_hostname }}-osd-dev%i
 ExecStart=/usr/bin/docker run --rm --net=host --pid=host\
    {% if not osd_containerized_deployment_with_kv -%}
    -v /var/lib/ceph:/var/lib/ceph \
@@ -13,6 +13,7 @@ ExecStart=/usr/bin/docker run --rm --net=host --pid=host\
    {% else -%}
    -e KV_TYPE={{kv_type}} \
    -e KV_IP={{kv_endpoint}} \
+   -e KV_PORT={{kv_port}} \
    {% endif -%}
    -v /dev:/dev \
    --privileged \