From 196fa7ef3912559b7d58c4a531910271d67d38c7 Mon Sep 17 00:00:00 2001
From: David Galloway <dgallowa@redhat.com>
Date: Thu, 11 May 2017 16:19:11 -0400
Subject: [PATCH 1/2] docker: Retry osd disk prep to workaround race condition

Fixes: https://github.com/ceph/ceph-ansible/issues/1495

Signed-off-by: David Galloway <dgallowa@redhat.com>
---
 roles/ceph-osd/tasks/docker/start_docker_osd.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
index 3423a9452..31c82ed64 100644
--- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml
+++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
@@ -41,6 +41,9 @@
     - item.1.get("rc", 0) != 0
     - ceph_osd_docker_prepare_env is defined
     - not osd_containerized_deployment_with_kv
+  # https://github.com/ceph/ceph-ansible/issues/1495
+  retries: 5
+  delay: 3
 
 - name: prepare ceph osd disk with kv_store
   shell: |
@@ -66,6 +69,9 @@
     - item.1.get("rc", 0) != 0
     - ceph_osd_docker_prepare_env is defined
     - osd_containerized_deployment_with_kv
+  # https://github.com/ceph/ceph-ansible/issues/1495
+  retries: 5
+  delay: 3
 
 - name: generate ceph osd docker run script
   become: true

From 1e7010de7f5f5070601218e6288dc94cbb8f6558 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@redhat.com>
Date: Mon, 15 May 2017 11:50:23 +0200
Subject: [PATCH 2/2] Docker: rm container before retry of ceph osd prepare

In addition to `196fa7e` this commit check if a container has been
already launched and delete it before retrying the ceph osd prepare
process.

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
---
 roles/ceph-osd/tasks/docker/start_docker_osd.yml | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
index 31c82ed64..2b0aad3bd 100644
--- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml
+++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
@@ -21,10 +21,12 @@
 # starting the next task
 - name: prepare ceph osd disk
   shell: |
+    NAME="ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item.0 | regex_replace('/', '') }}"
+    docker ps -a | grep -sq $NAME && docker rm $NAME || true
     docker run --net=host \
     --pid=host \
     --privileged=true \
-    --name="ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item.0 | regex_replace('/', '') }}" \
+    --name=$NAME \
     -v /etc/ceph:/etc/ceph \
     -v /var/lib/ceph/:/var/lib/ceph/ \
     -v /dev:/dev \
@@ -42,15 +44,19 @@
     - ceph_osd_docker_prepare_env is defined
     - not osd_containerized_deployment_with_kv
   # https://github.com/ceph/ceph-ansible/issues/1495
+  register: prepare_osd_disk
+  until: prepare_osd_disk.rc == 0
   retries: 5
   delay: 3
 
 - name: prepare ceph osd disk with kv_store
   shell: |
+    NAME="ceph-osd-prepare-{{ ansible_hostname }}-dev-{{ item.0 | regex_replace('/', '') }}"
+    docker ps -a | grep -sq $NAME && docker rm $NAME || true
     docker run  --net=host   \
     --pid=host \
     --privileged=true \
-    --name="ceph-osd-prepare-{{ ansible_hostname }}-dev-{{ item.0 | regex_replace('/', '') }}" \
+    --name=$NAME \
     -v /dev:/dev \
     -v /etc/localtime:/etc/localtime:ro \
     -e "OSD_DEVICE={{ item.0 }}" \
@@ -70,6 +76,8 @@
     - ceph_osd_docker_prepare_env is defined
     - osd_containerized_deployment_with_kv
   # https://github.com/ceph/ceph-ansible/issues/1495
+  register: prepare_osd_disk_kv
+  until: prepare_osd_disk_kv.rc == 0
   retries: 5
   delay: 3