Merge pull request #673 from ceph/docker-perm

ceph-docker: fix permissions on directories

roles/ceph-mds/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,43 @@
+---
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+  changed_when: false
+  failed_when: false
+
+# NOTE (leseb): we can not use docker inspect with 'format filed' because of
+# https://github.com/ansible/ansible/issues/10156
+- name: inspect ceph version
+  shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version.stdout not in ['firefly','giant', 'hammer']
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-mds
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "64045"
+    group: "64045"
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-mds
+  when: after_hamer

roles/ceph-mds/tasks/docker/main.yml

@@ -9,8 +9,10 @@
   when: ceph_health.rc != 0
 
 - include: pre_requisite.yml
+- include: fetch_configs.yml
+- include: dirs_permissions.yml
+
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'
 
-- include: fetch_configs.yml
 - include: start_docker_mds.yml

roles/ceph-mds/tasks/docker/pre_requisite.yml

@@ -1,12 +1,4 @@
 ---
-- name: create mds bootstrap directory
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - /etc/ceph/
-    - /var/lib/ceph/bootstrap-mds
-
 - name: install pip and docker on ubuntu
   apt:
     name: "{{ item }}"

roles/ceph-mon/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,47 @@
+---
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+  changed_when: false
+  failed_when: false
+
+# NOTE (leseb): we can not use docker inspect with 'format filed' because of
+# https://github.com/ansible/ansible/issues/10156
+- name: inspect ceph version
+  shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version.stdout not in ['firefly','giant', 'hammer']
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+    - /var/lib/ceph/bootstrap-mds
+    - /var/lib/ceph/bootstrap-rgw
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "64045"
+    group: "64045"
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+    - /var/lib/ceph/bootstrap-mds
+    - /var/lib/ceph/bootstrap-rgw
+  when: after_hamer

roles/ceph-mon/tasks/docker/main.yml

@@ -20,9 +20,6 @@
 
 - include: pre_requisite.yml
 
-- include: selinux.yml
-  when: ansible_os_family == 'RedHat'
-
 # let the first mon create configs and keyrings
 - include: create_configs.yml
   when:
@@ -32,6 +29,11 @@
 - include: fetch_configs.yml
   when: not mon_containerized_deployment_with_kv
 
+- include: dirs_permissions.yml
+
+- include: selinux.yml
+  when: ansible_os_family == 'RedHat'
+
 - include: start_docker_monitor.yml
 
 - include: copy_configs.yml

roles/ceph-mon/tasks/docker/pre_requisite.yml

@@ -1,14 +1,4 @@
 ---
-- name: create bootstrap directories
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - /etc/ceph/
-    - /var/lib/ceph/bootstrap-osd
-    - /var/lib/ceph/bootstrap-mds
-    - /var/lib/ceph/bootstrap-rgw
-
 - name: install pip and docker on ubuntu
   apt:
     name: "{{ item }}"

roles/ceph-mon/tasks/docker/start_docker_monitor.yml

@@ -1,7 +1,4 @@
 ---
-- name: pull ceph daemon image
-  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
-
 - name: populate kv_store with default ceph.conf
   docker:
     name: populate-kv-store
@@ -75,6 +72,8 @@
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
+  changed_when: false
+  failed_when: false
   when:
     is_atomic or
     ansible_os_family == 'CoreOS'

roles/ceph-osd/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,43 @@
+---
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+  changed_when: false
+  failed_when: false
+
+# NOTE (leseb): we can not use docker inspect with 'format filed' because of
+# https://github.com/ansible/ansible/issues/10156
+- name: inspect ceph version
+  shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version.stdout not in ['firefly','giant', 'hammer']
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "64045"
+    group: "64045"
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+  when: after_hamer

roles/ceph-osd/tasks/docker/main.yml

@@ -20,10 +20,12 @@
 
 - include: pre_requisite.yml
 
-- include: selinux.yml
-  when: ansible_os_family == 'RedHat'
-
 - include: fetch_configs.yml
   when: not osd_containerized_deployment_with_kv
 
+- include: dirs_permissions.yml
+
+- include: selinux.yml
+  when: ansible_os_family == 'RedHat'
+
 - include: start_docker_osd.yml

roles/ceph-osd/tasks/docker/pre_requisite.yml

@@ -1,12 +1,4 @@
 ---
-- name: create osd bootstrap directory
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - /etc/ceph/
-    - /var/lib/ceph/bootstrap-osd
-
 - name: install pip and docker on ubuntu
   apt:
     name: "{{ item }}"

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -9,9 +9,12 @@
     state: unmounted
   when: ceph_docker_on_openstack
 
-# (rootfs) for reasons I haven't figured out, docker pull and run will fail.
-- name: pull ceph daemon image
-  shell: "docker pull {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
+- name: verify if the disk was already prepared
+  shell: "lsblk -o PARTLABEL {{ item }} | grep -sq 'ceph'"
+  failed_when: false
+  changed_when: false
+  with_items: ceph_osd_docker_devices
+  register: osd_prepared
 
 - name: prepare ceph osd disk
   docker:
@@ -21,12 +24,17 @@
     pid: host
     state: running
     privileged: yes
-    env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_prepare_env }}"
-    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev/:/dev/"
-  with_items: ceph_osd_docker_devices
+    env: "OSD_DEVICE={{ item.0 }},{{ ceph_osd_docker_prepare_env }}"
+    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,{{ item.0 }}:{{ item.0 }}"
+  with_together:
+    - ceph_osd_docker_devices
+    - osd_prepared.results
   when:
+    item.1.get("rc", 0) != 0 and
     ceph_osd_docker_prepare_env is defined and
-    not osd_containerized_deployment_with_kv
+    not osd_containerized_deployment_with_kv and
+    not is_atomic and
+    not ansible_os_family == 'CoreOS'
 
 - name: prepare ceph osd disk with kv_store
   docker:
@@ -36,12 +44,63 @@
     pid: host
     state: running
     privileged: yes
-    volumes: "/dev/:/dev/"
-    env: "OSD_DEVICE={{ item }},KV_TYPE={{kv_type}},KV_IP={{kv_endpoint}},{{ ceph_osd_docker_prepare_env }}"
+    volumes: "{{ item.0 }}:{{ item.0 }}"
+    env: "OSD_DEVICE={{ item.0 }},KV_TYPE={{kv_type}},KV_IP={{kv_endpoint}},{{ ceph_osd_docker_prepare_env }}"
   ignore_errors: true
-  with_items: ceph_osd_docker_devices
+  with_together:
+    - ceph_osd_docker_devices
+    - osd_prepared.results
   when:
+    item.1.get("rc", 0) != 0 and
     ceph_osd_docker_prepare_env is defined and
+    osd_containerized_deployment_with_kv and
+    not is_atomic and
+    not ansible_os_family == 'CoreOS'
+
+me: prepare ceph osd disk for container operating systems
+  shell: |
+    docker run -d --net=host \
+    --pid=host \
+    --privileged=true \
+    -v /etc/ceph:/etc/ceph \
+    -v /var/lib/ceph/:/var/lib/ceph/ \
+    -v {{ item.0 }}:{{ item.0 }} \
+    -e OSD_DEVICE="{{ item.0 }}" \
+    -e "{{ ceph_osd_docker_prepare_env }}" \
+    "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" \
+    osd_ceph_disk_prepare
+  failed_when: false
+  changed_when: false
+  with_together:
+    - ceph_osd_docker_devices
+    - osd_prepared.results
+  when:
+    item.1.get("rc", 0) != 0 and
+    is_atomic or
+    ansible_os_family == 'CoreOS' and
+    not osd_containerized_deployment_with_kv
+
+- name: prepare ceph osd disk for container operating systems with kv_store
+  shell: |
+    docker run -d --net=host \
+    --pid=host \
+    --privileged=true \
+    -e KV_TYPE={{ kv_type }} \
+    -e KV_IP={{ kv_endpoint }} \
+    -v {{ item.0 }}:{{ item.0 }} \
+    -e OSD_DEVICE={{ item.0 }} \
+    -e "{{ ceph_osd_docker_prepare_env }}" \
+    "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" \
+    osd_ceph_disk_prepare
+  failed_when: false
+  changed_when: false
+  with_together:
+    - ceph_osd_docker_devices
+    - osd_prepared.results
+  when:
+    item.1.get("rc", 0) != 0 and
+    is_atomic or
+    ansible_os_family == 'CoreOS' and
     osd_containerized_deployment_with_kv
 
 # Use systemd to manage container on Atomic host
@@ -54,6 +113,9 @@
     group: "root"
     mode: "0644"
   failed_when: false
+  when:
+    is_atomic or
+    ansible_os_family == 'CoreOS'
 
 - name: link systemd unit file for osd instance
   file:
@@ -76,6 +138,8 @@
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
+  changed_when: false
+  failed_when: false
   when:
     is_atomic or
     ansible_os_family == 'CoreOS'
@@ -107,7 +171,6 @@
     ansible_os_family != 'CoreOS' and
     not osd_containerized_deployment_with_kv
 
-
 - name: run the ceph osd docker image with kv
   docker:
     image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"

roles/ceph-restapi/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,45 @@
+---
+- name: inspect ceph version
+  shell: "docker inspect --format '{{ index (index .Config.Env) 3 }}' docker.io/{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }} | cut -d '=' -f '2'"
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version not in (firefly or giant or hammer)
+
+- set_fact:
+    after_hamer=False
+  when:
+    ceph_version in (firefly or giant or hammer)
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+    - /var/lib/ceph/bootstrap-mds
+    - /var/lib/ceph/bootstrap-rgw
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: ceph
+    group: ceph
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-osd
+    - /var/lib/ceph/bootstrap-mds
+    - /var/lib/ceph/bootstrap-rgw
+  when: not after_hamer

roles/ceph-restapi/tasks/docker/main.yml

@@ -1,4 +1,5 @@
 ---
 - include: pre_requisite.yml
 - include: fetch_configs.yml
+- include: dirs_permissions.yml
 - include: start_docker_restapi.yml

roles/ceph-rgw/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,43 @@
+---
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+  changed_when: false
+  failed_when: false
+
+# NOTE (leseb): we can not use docker inspect with 'format filed' because of
+# https://github.com/ansible/ansible/issues/10156
+- name: inspect ceph version
+  shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hamer=True
+  when:
+    ceph_version.stdout not in ['firefly','giant', 'hammer']
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-rgw
+  when: not after_hamer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "64045"
+    group: "64045"
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /var/lib/ceph/bootstrap-rgw
+  when: after_hamer

roles/ceph-rgw/tasks/docker/main.yml

@@ -9,8 +9,10 @@
   when: ceph_health.rc != 0
 
 - include: pre_requisite.yml
+- include: fetch_configs.yml
+- include: dirs_permissions.yml
+
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'
 
-- include: fetch_configs.yml
 - include: start_docker_rgw.yml

roles/ceph-rgw/tasks/docker/pre_requisite.yml

@@ -1,12 +1,4 @@
 ---
-- name: create rgw bootstrap directory
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - /etc/ceph/
-    - /var/lib/ceph/bootstrap-rgw
-
 - name: install pip and docker on ubuntu
   apt:
     name: "{{ item }}"

roles/ceph-rgw/tasks/docker/start_docker_rgw.yml

@@ -1,7 +1,4 @@
 ---
-- name: pull ceph daemon image
-  shell: "docker pull {{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"
-
 - name: run the rados gateway docker image
   docker:
     image: "{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"