ceph-docker: fix permissions on directories

fixing the can't open /var/lib/ceph/bootstrap-osd/ceph.keyring: can't
open /var/lib/ceph/bootstrap-osd/ceph.keyring: (13) Permission denied

Signed-off-by: Sébastien Han <seb@redhat.com>

roles/ceph-mds/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,43 @@
+---
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+  changed_when: false
+  failed_when: false
+
+# NOTE (leseb): we can not use docker inspect with 'format filed' because of
+# https://github.com/ansible/ansible/issues/10156
+- name: inspect ceph version
+  shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version.stdout not in ['firefly','giant', 'hammer']
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-mds
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "64045"
+    group: "64045"
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-mds
+  when: after_hamer

roles/ceph-mds/tasks/docker/main.yml

@@ -9,8 +9,10 @@
   when: ceph_health.rc != 0
 
 - include: pre_requisite.yml
+- include: fetch_configs.yml
+- include: dirs_permissions.yml
+
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'
 
-- include: fetch_configs.yml
 - include: start_docker_mds.yml

roles/ceph-mds/tasks/docker/pre_requisite.yml

@@ -1,12 +1,4 @@
 ---
-- name: create mds bootstrap directory
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - /etc/ceph/
-    - /var/lib/ceph/bootstrap-mds
-
 - name: install pip and docker on ubuntu
   apt:
     name: "{{ item }}"

roles/ceph-mon/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,47 @@
+---
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+  changed_when: false
+  failed_when: false
+
+# NOTE (leseb): we can not use docker inspect with 'format filed' because of
+# https://github.com/ansible/ansible/issues/10156
+- name: inspect ceph version
+  shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version.stdout not in ['firefly','giant', 'hammer']
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+    - /var/lib/ceph/bootstrap-mds
+    - /var/lib/ceph/bootstrap-rgw
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "64045"
+    group: "64045"
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+    - /var/lib/ceph/bootstrap-mds
+    - /var/lib/ceph/bootstrap-rgw
+  when: after_hamer

roles/ceph-mon/tasks/docker/main.yml

@@ -20,9 +20,6 @@
 
 - include: pre_requisite.yml
 
-- include: selinux.yml
-  when: ansible_os_family == 'RedHat'
-
 # let the first mon create configs and keyrings
 - include: create_configs.yml
   when:
@@ -32,6 +29,11 @@
 - include: fetch_configs.yml
   when: not mon_containerized_deployment_with_kv
 
+- include: dirs_permissions.yml
+
+- include: selinux.yml
+  when: ansible_os_family == 'RedHat'
+
 - include: start_docker_monitor.yml
 
 - include: copy_configs.yml

roles/ceph-mon/tasks/docker/pre_requisite.yml

@@ -1,14 +1,4 @@
 ---
-- name: create bootstrap directories
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - /etc/ceph/
-    - /var/lib/ceph/bootstrap-osd
-    - /var/lib/ceph/bootstrap-mds
-    - /var/lib/ceph/bootstrap-rgw
-
 - name: install pip and docker on ubuntu
   apt:
     name: "{{ item }}"

roles/ceph-mon/tasks/docker/start_docker_monitor.yml

@@ -1,7 +1,4 @@
 ---
-- name: pull ceph daemon image
-  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
-
 - name: populate kv_store with default ceph.conf
   docker:
     name: populate-kv-store
@@ -75,6 +72,8 @@
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
+  changed_when: false
+  failed_when: false
   when:
     is_atomic or
     ansible_os_family == 'CoreOS'

roles/ceph-osd/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,43 @@
+---
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+  changed_when: false
+  failed_when: false
+
+# NOTE (leseb): we can not use docker inspect with 'format filed' because of
+# https://github.com/ansible/ansible/issues/10156
+- name: inspect ceph version
+  shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version.stdout not in ['firefly','giant', 'hammer']
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "64045"
+    group: "64045"
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+  when: after_hamer

roles/ceph-osd/tasks/docker/main.yml

@@ -20,10 +20,12 @@
 
 - include: pre_requisite.yml
 
-- include: selinux.yml
-  when: ansible_os_family == 'RedHat'
-
 - include: fetch_configs.yml
   when: not osd_containerized_deployment_with_kv
 
+- include: dirs_permissions.yml
+
+- include: selinux.yml
+  when: ansible_os_family == 'RedHat'
+
 - include: start_docker_osd.yml

roles/ceph-osd/tasks/docker/pre_requisite.yml

@@ -1,12 +1,4 @@
 ---
-- name: create osd bootstrap directory
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - /etc/ceph/
-    - /var/lib/ceph/bootstrap-osd
-
 - name: install pip and docker on ubuntu
   apt:
     name: "{{ item }}"

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -9,10 +9,6 @@
     state: unmounted
   when: ceph_docker_on_openstack
 
-# (rootfs) for reasons I haven't figured out, docker pull and run will fail.
-- name: pull ceph daemon image
-  shell: "docker pull {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
-
 - name: prepare ceph osd disk
   docker:
     image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
@@ -76,6 +72,8 @@
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
+  changed_when: false
+  failed_when: false
   when:
     is_atomic or
     ansible_os_family == 'CoreOS'
@@ -107,7 +105,6 @@
     ansible_os_family != 'CoreOS' and
     not osd_containerized_deployment_with_kv
 
-
 - name: run the ceph osd docker image with kv
   docker:
     image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"

roles/ceph-restapi/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,45 @@
+---
+- name: inspect ceph version
+  shell: "docker inspect --format '{{ index (index .Config.Env) 3 }}' docker.io/{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }} | cut -d '=' -f '2'"
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version not in (firefly or giant or hammer)
+
+- set_fact:
+    after_hamer=False
+  when:
+    ceph_version in (firefly or giant or hammer)
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+    - /var/lib/ceph/bootstrap-mds
+    - /var/lib/ceph/bootstrap-rgw
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: ceph
+    group: ceph
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+    - /var/lib/ceph/bootstrap-mds
+    - /var/lib/ceph/bootstrap-rgw
+  when: not after_hamer

roles/ceph-restapi/tasks/docker/main.yml

@@ -1,4 +1,5 @@
 ---
 - include: pre_requisite.yml
 - include: fetch_configs.yml
+- include: dirs_permissions.yml
 - include: start_docker_restapi.yml

roles/ceph-rgw/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,43 @@
+---
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+  changed_when: false
+  failed_when: false
+
+# NOTE (leseb): we can not use docker inspect with 'format filed' because of
+# https://github.com/ansible/ansible/issues/10156
+- name: inspect ceph version
+  shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version.stdout not in ['firefly','giant', 'hammer']
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-rgw
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "64045"
+    group: "64045"
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-rgw
+  when: after_hamer

roles/ceph-rgw/tasks/docker/main.yml

@@ -9,8 +9,10 @@
   when: ceph_health.rc != 0
 
 - include: pre_requisite.yml
+- include: fetch_configs.yml
+- include: dirs_permissions.yml
+
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'
 
-- include: fetch_configs.yml
 - include: start_docker_rgw.yml

roles/ceph-rgw/tasks/docker/pre_requisite.yml

@@ -1,12 +1,4 @@
 ---
-- name: create rgw bootstrap directory
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - /etc/ceph/
-    - /var/lib/ceph/bootstrap-rgw
-
 - name: install pip and docker on ubuntu
   apt:
     name: "{{ item }}"

roles/ceph-rgw/tasks/docker/start_docker_rgw.yml

@@ -1,7 +1,4 @@
 ---
-- name: pull ceph daemon image
-  shell: "docker pull {{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"
-
 - name: run the rados gateway docker image
   docker:
     image: "{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"