From ab7eb79212f90edbfe29faf40dac5d209c7a70a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 2 Nov 2017 14:53:46 +0100 Subject: [PATCH 1/2] config: fix monitor_interface when not passed in the inventory file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting monitor_interface in group_vars/all.yml makes the hostvars[host]['monitor_interface'] non-existing. Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1507922 Signed-off-by: Sébastien Han --- roles/ceph-config/templates/ceph.conf.j2 | 23 ++++++++-- roles/ceph-mon/templates/ceph-mon.service.j2 | 44 +++++++++++--------- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/roles/ceph-config/templates/ceph.conf.j2 b/roles/ceph-config/templates/ceph.conf.j2 index ef9046535..77ac36c1f 100644 --- a/roles/ceph-config/templates/ceph.conf.j2 +++ b/roles/ceph-config/templates/ceph.conf.j2 @@ -50,11 +50,19 @@ mon host = {% if nb_mon > 0 %} {%- elif ip_version == 'ipv6' -%} [{{ hostvars[host]['monitor_address'] }}] {%- endif %} - {%- else -%} + {%- elif hostvars[host]['monitor_interface'] is defined -%} + {% set interface = 'ansible_' + hostvars[host]['monitor_interface'] %} {% if ip_version == 'ipv4' -%} - {{ hostvars[host]['ansible_' + hostvars[host]['monitor_interface']][ip_version]['address'] }} + {{ hostvars[host][interface][ip_version]['address'] }} {%- elif ip_version == 'ipv6' -%} - [{{ hostvars[host]['ansible_' + hostvars[host]['monitor_interface']][ip_version][0]['address'] }}] + [{{ hostvars[host][interface][ip_version][0]['address'] }}] + {%- endif %} + {%- else -%} + {% set interface = 'ansible_' + monitor_interface %} + {% if ip_version == 'ipv4' -%} + {{ hostvars[host][interface][ip_version]['address'] }} + {%- elif ip_version == 'ipv6' -%} + [{{ hostvars[host][interface][ip_version][0]['address'] }}] {%- endif %} {%- endif %} {% if not loop.last -%},{%- endif %} @@ -80,13 +88,20 @@ mon host = {% if nb_mon > 0 %} {%- elif ip_version == 'ipv6' -%} [{{ hostvars[host]['monitor_address'] }}] {%- endif %} - {%- else -%} + {%- elif hostvars[host]['monitor_interface'] is defined -%} {% set interface = 'ansible_' + hostvars[host]['monitor_interface'] %} {% if ip_version == 'ipv4' -%} {{ hostvars[host][interface][ip_version]['address'] }} {%- elif ip_version == 'ipv6' -%} [{{ hostvars[host][interface][ip_version][0]['address'] }}] {%- endif %} + {%- else -%} + {% set interface = 'ansible_' + monitor_interface %} + {% if ip_version == 'ipv4' -%} + {{ [interface][ip_version]['address'] }} + {%- elif ip_version == 'ipv6' -%} + [{{ [interface][ip_version][0]['address'] }}] + {%- endif %} {%- endif %} {% if not loop.last -%},{%- endif %} {%- endfor %} diff --git a/roles/ceph-mon/templates/ceph-mon.service.j2 b/roles/ceph-mon/templates/ceph-mon.service.j2 index d7a99e49b..7da8bb043 100644 --- a/roles/ceph-mon/templates/ceph-mon.service.j2 +++ b/roles/ceph-mon/templates/ceph-mon.service.j2 @@ -8,48 +8,54 @@ ExecStartPre=-/usr/bin/docker rm ceph-mon-%i ExecStartPre=$(command -v mkdir) -p /etc/ceph /var/lib/ceph/mon ExecStart=/usr/bin/docker run --rm --name ceph-mon-%i --net=host \ --memory={{ ceph_mon_docker_memory_limit }} \ - {% if docker_version.split('.')[0] | version_compare('13', '>=') -%} +{% if docker_version.split('.')[0] | version_compare('13', '>=') -%} --cpus={{ ceph_mon_docker_cpu_limit }} \ - {% else -%} +{% else -%} --cpu-quota={{ ceph_mon_docker_cpu_limit * 100000 }} \ - {% endif -%} - {% if not containerized_deployment_with_kv -%} +{% endif -%} +{% if not containerized_deployment_with_kv -%} -v /var/lib/ceph:/var/lib/ceph \ -v /etc/ceph:/etc/ceph \ - {% else -%} +{% else -%} -e KV_TYPE={{kv_type}} \ -e KV_IP={{kv_endpoint}}\ -e KV_PORT={{kv_port}} \ - {% endif -%} +{% endif -%} -v /etc/localtime:/etc/localtime:ro \ - {% if mon_docker_privileged -%} +{% if mon_docker_privileged -%} --privileged \ - {% endif -%} - {% if mon_docker_net_host -%} +{% endif -%} +{% if mon_docker_net_host -%} --net=host \ - {% endif -%} +{% endif -%} -e IP_VERSION={{ ip_version[-1:] }} \ - {% if monitor_address_block != 'subnet' %} +{% if monitor_address_block != 'subnet' %} {% if ip_version == 'ipv4' -%} -e MON_IP={{ hostvars[inventory_hostname]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }} \ {% elif ip_version == 'ipv6' -%} -e MON_IP=[{{ hostvars[inventory_hostname]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }}] \ {% endif -%} - {% elif hostvars[inventory_hostname]['monitor_address'] is defined and hostvars[inventory_hostname]['monitor_address'] != '0.0.0.0' -%} +{% elif hostvars[inventory_hostname]['monitor_address'] is defined and hostvars[inventory_hostname]['monitor_address'] != '0.0.0.0' %} {% if ip_version == 'ipv4' -%} -e MON_IP={{ hostvars[inventory_hostname]['monitor_address'] }} \ {% elif ip_version == 'ipv6' -%} -e MON_IP=[{{ hostvars[inventory_hostname]['monitor_address'] }}] \ {% endif -%} - {% else -%} +{% elif hostvars[inventory_hostname]['monitor_interface'] is defined %} {% set interface = 'ansible_' + hostvars[inventory_hostname]['monitor_interface'] %} - {% if ip_version == 'ipv6' -%} - -e MON_IP=[{{ hostvars[inventory_hostname][interface][ip_version][0]['address'] }}] \ - {% elif ip_version == 'ipv4' -%} + {% if ip_version == 'ipv4' -%} -e MON_IP={{ hostvars[inventory_hostname][interface][ip_version]['address'] }} \ - {% endif -%} - {% endif -%} - + {% elif ip_version == 'ipv6' -%} + -e MON_IP=[{{ hostvars[inventory_hostname][interface][ip_version][0]['address'] }}] \ + {% endif -%} +{% else %} + {% set interface = 'ansible_' + monitor_interface %} + {% if ip_version == 'ipv4' -%} + -e MON_IP={{ [interface][ip_version]['address'] }} \ + {% elif ip_version == 'ipv6' -%} + -e MON_IP=[{{ [interface][ip_version][0]['address'] }}] \ + {% endif -%} +{%- endif %} -e CLUSTER={{ cluster }} \ -e FSID={{ fsid }} \ -e CEPH_PUBLIC_NETWORK={{ public_network }} \ From d4ed9a2064e503ac4a4fe978cb9e196ca9150272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 2 Nov 2017 16:17:38 +0100 Subject: [PATCH 2/2] osd: enhance backward compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During the initial implementation of this 'old' thing we were falling into this issue without noticing https://github.com/moby/moby/issues/30341 and where blindly using --rm, now this is fixed the prepare container disappears and thus activation fail. I'm fixing this for old jewel images. Also this fixes the machine reboot case where the docker logs are purgend. In the old scenario, we now store the log locally in the same directory as the ceph-osd-run.sh script. Signed-off-by: Sébastien Han --- roles/ceph-client/tasks/create_users_keys.yml | 2 +- roles/ceph-docker-common/tasks/main.yml | 2 +- .../tasks/docker/start_docker_osd.yml | 2 +- roles/ceph-osd/tasks/scenarios/collocated.yml | 2 -- .../tasks/scenarios/non-collocated.yml | 2 -- roles/ceph-osd/templates/ceph-osd-run.sh.j2 | 21 +++++++++++++++---- 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/roles/ceph-client/tasks/create_users_keys.yml b/roles/ceph-client/tasks/create_users_keys.yml index 96d64c513..47fd2d4e6 100644 --- a/roles/ceph-client/tasks/create_users_keys.yml +++ b/roles/ceph-client/tasks/create_users_keys.yml @@ -6,7 +6,7 @@ - name: set docker_exec_client_cmd for containers set_fact: - docker_exec_client_cmd: docker run -v /etc/ceph:/etc/ceph --entrypoint /usr/bin/{{ docker_exec_client_cmd_binary }} {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} + docker_exec_client_cmd: docker run --rm -v /etc/ceph:/etc/ceph --entrypoint /usr/bin/{{ docker_exec_client_cmd_binary }} {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} when: containerized_deployment - name: set docker_exec_client_cmd for non-containers diff --git a/roles/ceph-docker-common/tasks/main.yml b/roles/ceph-docker-common/tasks/main.yml index d3752181a..195711107 100644 --- a/roles/ceph-docker-common/tasks/main.yml +++ b/roles/ceph-docker-common/tasks/main.yml @@ -94,7 +94,7 @@ - fetch_container_image - name: get ceph version - command: docker run --entrypoint /usr/bin/ceph {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} --version + command: docker run --rm --entrypoint /usr/bin/ceph {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} --version changed_when: false check_mode: no register: ceph_version diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index c038e9164..ec62f24eb 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -11,7 +11,7 @@ - ceph_docker_on_openstack - name: test if the container image has the disk_list function - command: docker run --entrypoint=stat {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} disk_list.sh + command: docker run --rm --entrypoint=stat {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} disk_list.sh changed_when: false failed_when: false register: disk_list diff --git a/roles/ceph-osd/tasks/scenarios/collocated.yml b/roles/ceph-osd/tasks/scenarios/collocated.yml index 6c12c8912..9c6621aff 100644 --- a/roles/ceph-osd/tasks/scenarios/collocated.yml +++ b/roles/ceph-osd/tasks/scenarios/collocated.yml @@ -5,7 +5,6 @@ - name: prepare ceph containerized osd disk collocated shell: | docker run --net=host \ - --rm \ --pid=host \ --privileged=true \ --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \ @@ -32,7 +31,6 @@ - name: automatic prepare ceph containerized osd disk collocated shell: | docker run --net=host \ - --rm \ --pid=host \ --privileged=true \ --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.key }} \ diff --git a/roles/ceph-osd/tasks/scenarios/non-collocated.yml b/roles/ceph-osd/tasks/scenarios/non-collocated.yml index ef4d49677..9b8c73f57 100644 --- a/roles/ceph-osd/tasks/scenarios/non-collocated.yml +++ b/roles/ceph-osd/tasks/scenarios/non-collocated.yml @@ -5,7 +5,6 @@ - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated shell: | docker run --net=host \ - --rm \ --pid=host \ --privileged=true \ --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \ @@ -34,7 +33,6 @@ - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated with a dedicated device for db and wal shell: | docker run --net=host \ - --rm \ --pid=host \ --privileged=true \ --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \ diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 index 6c6bf2274..724d4bfb9 100644 --- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 +++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 @@ -7,7 +7,7 @@ ############# {% if disk_list.get("rc", 1) == 0 -%} function expose_partitions () { -DOCKER_ENV=$(docker run --name expose_partitions_${1} --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph -e CLUSTER={{ cluster }} -e OSD_DEVICE=/dev/${1} {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} disk_list) +DOCKER_ENV=$(docker run --rm --name expose_partitions_${1} --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph -e CLUSTER={{ cluster }} -e OSD_DEVICE=/dev/${1} {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} disk_list) docker rm -f expose_partitions_${1} } {% else -%} @@ -15,9 +15,22 @@ DOCKER_ENV=$(docker run --name expose_partitions_${1} --privileged=true -v /dev/ # Jewel images prior to https://github.com/ceph/ceph-docker/pull/797 REGEX="[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" function expose_partitions { - local partition - if docker logs ceph-osd-prepare-{{ ansible_hostname }}-devdev${1} |& grep -Eo "$partition is GPT partition"; then - part=$(docker logs ceph-osd-prepare-{{ ansible_hostname }}-devdev${1} |& grep "$partition is GPT partition" | grep -Eo /dev/disk/by-partuuid/${REGEX} | uniq) + if docker ps -a | grep -sq ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}; then + if [[ ! -f {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}.log ]]; then + docker logs ceph-osd-prepare-{{ ansible_hostname }}-devdev${1} &> {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}.log + fi + fi + if docker ps -a | grep -sq ceph-osd-prepare-{{ ansible_hostname }}-${1}; then + if [[ ! -f {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-${1}.log ]]; then + docker logs ceph-osd-prepare-{{ ansible_hostname }}-${1} &> {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-${1}.log + fi + fi + if [[ -f {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}.log ]]; then + part=$(grep "Journal is GPT partition" {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}.log | grep -Eo /dev/disk/by-partuuid/${REGEX} | uniq) + DOCKER_ENV="$DOCKER_ENV -e OSD_JOURNAL=$part" + fi + if [[ -f {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-${1}.log ]]; then + part=$(grep "Journal is GPT partition" {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-${1}.log | grep -Eo /dev/disk/by-partuuid/${REGEX} | uniq) DOCKER_ENV="$DOCKER_ENV -e OSD_JOURNAL=$part" fi }