diff --git a/roles/ceph-mds/tasks/docker/dirs_permissions.yml b/roles/ceph-mds/tasks/docker/dirs_permissions.yml new file mode 100644 index 000000000..ba5818294 --- /dev/null +++ b/roles/ceph-mds/tasks/docker/dirs_permissions.yml @@ -0,0 +1,43 @@ +--- +- name: pull ceph daemon image + shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" + changed_when: false + failed_when: false + +# NOTE (leseb): we can not use docker inspect with 'format filed' because of +# https://github.com/ansible/ansible/issues/10156 +- name: inspect ceph version + shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq + changed_when: false + failed_when: false + run_once: true + register: ceph_version + +- set_fact: + after_hamer=True + when: + ceph_version.stdout not in ['firefly','giant', 'hammer'] + +- name: create bootstrap directories (for or before hammer) + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-mds + when: not after_hamer + +- name: create bootstrap directories (after hammer) + file: + path: "{{ item }}" + state: directory + owner: "64045" + group: "64045" + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-mds + when: after_hamer diff --git a/roles/ceph-mds/tasks/docker/main.yml b/roles/ceph-mds/tasks/docker/main.yml index 1c92d4845..8750a8259 100644 --- a/roles/ceph-mds/tasks/docker/main.yml +++ b/roles/ceph-mds/tasks/docker/main.yml @@ -9,8 +9,10 @@ when: ceph_health.rc != 0 - include: pre_requisite.yml +- include: fetch_configs.yml +- include: dirs_permissions.yml + - include: selinux.yml when: ansible_os_family == 'RedHat' -- include: fetch_configs.yml - include: start_docker_mds.yml diff --git a/roles/ceph-mds/tasks/docker/pre_requisite.yml b/roles/ceph-mds/tasks/docker/pre_requisite.yml index 69801ef77..bfa8d7a72 100644 --- a/roles/ceph-mds/tasks/docker/pre_requisite.yml +++ b/roles/ceph-mds/tasks/docker/pre_requisite.yml @@ -1,12 +1,4 @@ --- -- name: create mds bootstrap directory - file: - path: "{{ item }}" - state: directory - with_items: - - /etc/ceph/ - - /var/lib/ceph/bootstrap-mds - - name: install pip and docker on ubuntu apt: name: "{{ item }}" diff --git a/roles/ceph-mon/tasks/docker/dirs_permissions.yml b/roles/ceph-mon/tasks/docker/dirs_permissions.yml new file mode 100644 index 000000000..a33a38edf --- /dev/null +++ b/roles/ceph-mon/tasks/docker/dirs_permissions.yml @@ -0,0 +1,47 @@ +--- +- name: pull ceph daemon image + shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" + changed_when: false + failed_when: false + +# NOTE (leseb): we can not use docker inspect with 'format filed' because of +# https://github.com/ansible/ansible/issues/10156 +- name: inspect ceph version + shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq + changed_when: false + failed_when: false + run_once: true + register: ceph_version + +- set_fact: + after_hamer=True + when: + ceph_version.stdout not in ['firefly','giant', 'hammer'] + +- name: create bootstrap directories (for or before hammer) + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-osd + - /var/lib/ceph/bootstrap-mds + - /var/lib/ceph/bootstrap-rgw + when: not after_hamer + +- name: create bootstrap directories (after hammer) + file: + path: "{{ item }}" + state: directory + owner: "64045" + group: "64045" + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-osd + - /var/lib/ceph/bootstrap-mds + - /var/lib/ceph/bootstrap-rgw + when: after_hamer diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index ea69c32c3..960581b7e 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -20,9 +20,6 @@ - include: pre_requisite.yml -- include: selinux.yml - when: ansible_os_family == 'RedHat' - # let the first mon create configs and keyrings - include: create_configs.yml when: @@ -32,6 +29,11 @@ - include: fetch_configs.yml when: not mon_containerized_deployment_with_kv +- include: dirs_permissions.yml + +- include: selinux.yml + when: ansible_os_family == 'RedHat' + - include: start_docker_monitor.yml - include: copy_configs.yml diff --git a/roles/ceph-mon/tasks/docker/pre_requisite.yml b/roles/ceph-mon/tasks/docker/pre_requisite.yml index f09655f64..4459aa5b3 100644 --- a/roles/ceph-mon/tasks/docker/pre_requisite.yml +++ b/roles/ceph-mon/tasks/docker/pre_requisite.yml @@ -1,14 +1,4 @@ --- -- name: create bootstrap directories - file: - path: "{{ item }}" - state: directory - with_items: - - /etc/ceph/ - - /var/lib/ceph/bootstrap-osd - - /var/lib/ceph/bootstrap-mds - - /var/lib/ceph/bootstrap-rgw - - name: install pip and docker on ubuntu apt: name: "{{ item }}" diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 713f9bfec..908894ad9 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -1,7 +1,4 @@ --- -- name: pull ceph daemon image - shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" - - name: populate kv_store with default ceph.conf docker: name: populate-kv-store @@ -75,6 +72,8 @@ - name: reload systemd unit files shell: systemctl daemon-reload + changed_when: false + failed_when: false when: is_atomic or ansible_os_family == 'CoreOS' diff --git a/roles/ceph-osd/tasks/docker/dirs_permissions.yml b/roles/ceph-osd/tasks/docker/dirs_permissions.yml new file mode 100644 index 000000000..d0c49cf9c --- /dev/null +++ b/roles/ceph-osd/tasks/docker/dirs_permissions.yml @@ -0,0 +1,43 @@ +--- +- name: pull ceph daemon image + shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" + changed_when: false + failed_when: false + +# NOTE (leseb): we can not use docker inspect with 'format filed' because of +# https://github.com/ansible/ansible/issues/10156 +- name: inspect ceph version + shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq + changed_when: false + failed_when: false + run_once: true + register: ceph_version + +- set_fact: + after_hamer=True + when: + ceph_version.stdout not in ['firefly','giant', 'hammer'] + +- name: create bootstrap directories (for or before hammer) + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-osd + when: not after_hamer + +- name: create bootstrap directories (after hammer) + file: + path: "{{ item }}" + state: directory + owner: "64045" + group: "64045" + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-osd + when: after_hamer diff --git a/roles/ceph-osd/tasks/docker/main.yml b/roles/ceph-osd/tasks/docker/main.yml index b00d95138..7da6129bb 100644 --- a/roles/ceph-osd/tasks/docker/main.yml +++ b/roles/ceph-osd/tasks/docker/main.yml @@ -20,10 +20,12 @@ - include: pre_requisite.yml -- include: selinux.yml - when: ansible_os_family == 'RedHat' - - include: fetch_configs.yml when: not osd_containerized_deployment_with_kv +- include: dirs_permissions.yml + +- include: selinux.yml + when: ansible_os_family == 'RedHat' + - include: start_docker_osd.yml diff --git a/roles/ceph-osd/tasks/docker/pre_requisite.yml b/roles/ceph-osd/tasks/docker/pre_requisite.yml index a4837f11f..4459aa5b3 100644 --- a/roles/ceph-osd/tasks/docker/pre_requisite.yml +++ b/roles/ceph-osd/tasks/docker/pre_requisite.yml @@ -1,12 +1,4 @@ --- -- name: create osd bootstrap directory - file: - path: "{{ item }}" - state: directory - with_items: - - /etc/ceph/ - - /var/lib/ceph/bootstrap-osd - - name: install pip and docker on ubuntu apt: name: "{{ item }}" diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index 41c2acf63..929d6a0dd 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -9,9 +9,12 @@ state: unmounted when: ceph_docker_on_openstack -# (rootfs) for reasons I haven't figured out, docker pull and run will fail. -- name: pull ceph daemon image - shell: "docker pull {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" +- name: verify if the disk was already prepared + shell: "lsblk -o PARTLABEL {{ item }} | grep -sq 'ceph'" + failed_when: false + changed_when: false + with_items: ceph_osd_docker_devices + register: osd_prepared - name: prepare ceph osd disk docker: @@ -21,12 +24,17 @@ pid: host state: running privileged: yes - env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_prepare_env }}" - volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev/:/dev/" - with_items: ceph_osd_docker_devices + env: "OSD_DEVICE={{ item.0 }},{{ ceph_osd_docker_prepare_env }}" + volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,{{ item.0 }}:{{ item.0 }}" + with_together: + - ceph_osd_docker_devices + - osd_prepared.results when: + item.1.get("rc", 0) != 0 and ceph_osd_docker_prepare_env is defined and - not osd_containerized_deployment_with_kv + not osd_containerized_deployment_with_kv and + not is_atomic and + not ansible_os_family == 'CoreOS' - name: prepare ceph osd disk with kv_store docker: @@ -36,12 +44,63 @@ pid: host state: running privileged: yes - volumes: "/dev/:/dev/" - env: "OSD_DEVICE={{ item }},KV_TYPE={{kv_type}},KV_IP={{kv_endpoint}},{{ ceph_osd_docker_prepare_env }}" + volumes: "{{ item.0 }}:{{ item.0 }}" + env: "OSD_DEVICE={{ item.0 }},KV_TYPE={{kv_type}},KV_IP={{kv_endpoint}},{{ ceph_osd_docker_prepare_env }}" ignore_errors: true - with_items: ceph_osd_docker_devices + with_together: + - ceph_osd_docker_devices + - osd_prepared.results when: + item.1.get("rc", 0) != 0 and ceph_osd_docker_prepare_env is defined and + osd_containerized_deployment_with_kv and + not is_atomic and + not ansible_os_family == 'CoreOS' + +me: prepare ceph osd disk for container operating systems + shell: | + docker run -d --net=host \ + --pid=host \ + --privileged=true \ + -v /etc/ceph:/etc/ceph \ + -v /var/lib/ceph/:/var/lib/ceph/ \ + -v {{ item.0 }}:{{ item.0 }} \ + -e OSD_DEVICE="{{ item.0 }}" \ + -e "{{ ceph_osd_docker_prepare_env }}" \ + "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" \ + osd_ceph_disk_prepare + failed_when: false + changed_when: false + with_together: + - ceph_osd_docker_devices + - osd_prepared.results + when: + item.1.get("rc", 0) != 0 and + is_atomic or + ansible_os_family == 'CoreOS' and + not osd_containerized_deployment_with_kv + +- name: prepare ceph osd disk for container operating systems with kv_store + shell: | + docker run -d --net=host \ + --pid=host \ + --privileged=true \ + -e KV_TYPE={{ kv_type }} \ + -e KV_IP={{ kv_endpoint }} \ + -v {{ item.0 }}:{{ item.0 }} \ + -e OSD_DEVICE={{ item.0 }} \ + -e "{{ ceph_osd_docker_prepare_env }}" \ + "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" \ + osd_ceph_disk_prepare + failed_when: false + changed_when: false + with_together: + - ceph_osd_docker_devices + - osd_prepared.results + when: + item.1.get("rc", 0) != 0 and + is_atomic or + ansible_os_family == 'CoreOS' and osd_containerized_deployment_with_kv # Use systemd to manage container on Atomic host @@ -54,6 +113,9 @@ group: "root" mode: "0644" failed_when: false + when: + is_atomic or + ansible_os_family == 'CoreOS' - name: link systemd unit file for osd instance file: @@ -76,6 +138,8 @@ - name: reload systemd unit files shell: systemctl daemon-reload + changed_when: false + failed_when: false when: is_atomic or ansible_os_family == 'CoreOS' @@ -107,7 +171,6 @@ ansible_os_family != 'CoreOS' and not osd_containerized_deployment_with_kv - - name: run the ceph osd docker image with kv docker: image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" diff --git a/roles/ceph-restapi/tasks/docker/dirs_permissions.yml b/roles/ceph-restapi/tasks/docker/dirs_permissions.yml new file mode 100644 index 000000000..99dbe4094 --- /dev/null +++ b/roles/ceph-restapi/tasks/docker/dirs_permissions.yml @@ -0,0 +1,45 @@ +--- +- name: inspect ceph version + shell: "docker inspect --format '{{ index (index .Config.Env) 3 }}' docker.io/{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }} | cut -d '=' -f '2'" + changed_when: false + failed_when: false + run_once: true + register: ceph_version + +- set_fact: + after_hamer=True + when: + ceph_version not in (firefly or giant or hammer) + +- set_fact: + after_hamer=False + when: + ceph_version in (firefly or giant or hammer) + +- name: create bootstrap directories (for or before hammer) + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-osd + - /var/lib/ceph/bootstrap-mds + - /var/lib/ceph/bootstrap-rgw + when: not after_hamer + +- name: create bootstrap directories (after hammer) + file: + path: "{{ item }}" + state: directory + owner: ceph + group: ceph + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-osd + - /var/lib/ceph/bootstrap-mds + - /var/lib/ceph/bootstrap-rgw + when: not after_hamer diff --git a/roles/ceph-restapi/tasks/docker/main.yml b/roles/ceph-restapi/tasks/docker/main.yml index 7210a3f5b..b54b12b5a 100644 --- a/roles/ceph-restapi/tasks/docker/main.yml +++ b/roles/ceph-restapi/tasks/docker/main.yml @@ -1,4 +1,5 @@ --- - include: pre_requisite.yml - include: fetch_configs.yml +- include: dirs_permissions.yml - include: start_docker_restapi.yml diff --git a/roles/ceph-rgw/tasks/docker/dirs_permissions.yml b/roles/ceph-rgw/tasks/docker/dirs_permissions.yml new file mode 100644 index 000000000..d4046e616 --- /dev/null +++ b/roles/ceph-rgw/tasks/docker/dirs_permissions.yml @@ -0,0 +1,43 @@ +--- +- name: pull ceph daemon image + shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" + changed_when: false + failed_when: false + +# NOTE (leseb): we can not use docker inspect with 'format filed' because of +# https://github.com/ansible/ansible/issues/10156 +- name: inspect ceph version + shell: docker inspect docker.io/ceph/daemon | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq + changed_when: false + failed_when: false + run_once: true + register: ceph_version + +- set_fact: + after_hamer=True + when: + ceph_version.stdout not in ['firefly','giant', 'hammer'] + +- name: create bootstrap directories (for or before hammer) + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-rgw + when: not after_hamer + +- name: create bootstrap directories (after hammer) + file: + path: "{{ item }}" + state: directory + owner: "64045" + group: "64045" + mode: "0755" + with_items: + - /etc/ceph/ + - /var/lib/ceph/bootstrap-rgw + when: after_hamer diff --git a/roles/ceph-rgw/tasks/docker/main.yml b/roles/ceph-rgw/tasks/docker/main.yml index ffd5db2e6..11b85d558 100644 --- a/roles/ceph-rgw/tasks/docker/main.yml +++ b/roles/ceph-rgw/tasks/docker/main.yml @@ -9,8 +9,10 @@ when: ceph_health.rc != 0 - include: pre_requisite.yml +- include: fetch_configs.yml +- include: dirs_permissions.yml + - include: selinux.yml when: ansible_os_family == 'RedHat' -- include: fetch_configs.yml - include: start_docker_rgw.yml diff --git a/roles/ceph-rgw/tasks/docker/pre_requisite.yml b/roles/ceph-rgw/tasks/docker/pre_requisite.yml index d288e6f58..f2d9ecd70 100644 --- a/roles/ceph-rgw/tasks/docker/pre_requisite.yml +++ b/roles/ceph-rgw/tasks/docker/pre_requisite.yml @@ -1,12 +1,4 @@ --- -- name: create rgw bootstrap directory - file: - path: "{{ item }}" - state: directory - with_items: - - /etc/ceph/ - - /var/lib/ceph/bootstrap-rgw - - name: install pip and docker on ubuntu apt: name: "{{ item }}" diff --git a/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml b/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml index ce551848a..f4fa89a6d 100644 --- a/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml +++ b/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml @@ -1,7 +1,4 @@ --- -- name: pull ceph daemon image - shell: "docker pull {{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}" - - name: run the rados gateway docker image docker: image: "{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"