From 8df9791bd34f55948be7d11349b891d7f98995eb Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Mon, 19 Sep 2022 05:20:23 +0200 Subject: [PATCH] osd: do not use ceph/daemon entrypoint This changes the entrypoint used for ceph-osd containerized daemons in the systemd template. Signed-off-by: Guillaume Abrioux --- .../tasks/prerequisites.yml | 8 +++ roles/ceph-osd/tasks/start_osds.yml | 10 +++ roles/ceph-osd/templates/ceph-osd.service.j2 | 45 +----------- roles/ceph-osd/templates/systemd-run.j2 | 71 +++++++++++++++++++ 4 files changed, 91 insertions(+), 43 deletions(-) create mode 100644 roles/ceph-osd/templates/systemd-run.j2 diff --git a/roles/ceph-container-common/tasks/prerequisites.yml b/roles/ceph-container-common/tasks/prerequisites.yml index ebb0bfcc3..c29d225cb 100644 --- a/roles/ceph-container-common/tasks/prerequisites.yml +++ b/roles/ceph-container-common/tasks/prerequisites.yml @@ -42,3 +42,11 @@ or inventory_hostname in groups.get(rgw_group_name, []) command: /usr/sbin/restorecon -RF /etc/pki/ca-trust/extracted changed_when: false + +- name: install python3 on osd nodes + package: + name: python3 + state: present + when: + - inventory_hostname in groups.get(osd_group_name, []) + - ansible_facts['os_family'] == 'RedHat' \ No newline at end of file diff --git a/roles/ceph-osd/tasks/start_osds.yml b/roles/ceph-osd/tasks/start_osds.yml index 9eb4d8c50..6a9dd46c6 100644 --- a/roles/ceph-osd/tasks/start_osds.yml +++ b/roles/ceph-osd/tasks/start_osds.yml @@ -48,6 +48,16 @@ group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}" with_items: "{{ ((ceph_osd_ids.stdout | default('{}', True) | from_json).keys() | list) | union(osd_ids_non_container.stdout_lines | default([])) }}" +- name: write /var/lib/ceph/osd/{{ cluster }}-{{ osd_id }}/run + template: + src: systemd-run.j2 + dest: "/var/lib/ceph/osd/{{ cluster }}-{{ item }}/run" + mode: "0700" + owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}" + group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}" + with_items: "{{ ((ceph_osd_ids.stdout | default('{}', True) | from_json).keys() | list) | union(osd_ids_non_container.stdout_lines | default([])) }}" + when: containerized_deployment | bool + - name: systemd start osd systemd: name: ceph-osd@{{ item }} diff --git a/roles/ceph-osd/templates/ceph-osd.service.j2 b/roles/ceph-osd/templates/ceph-osd.service.j2 index 9ea33bc77..de4dd9e11 100644 --- a/roles/ceph-osd/templates/ceph-osd.service.j2 +++ b/roles/ceph-osd/templates/ceph-osd.service.j2 @@ -17,53 +17,12 @@ EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/rm -f /%t/%n-pid /%t/%n-cid ExecStartPre=-/usr/bin/{{ container_binary }} rm --storage ceph-osd-%i ExecStartPre=-/usr/bin/mkdir -p /var/log/ceph +ExecStartPre=-/usr/bin/mkdir -p /var/lib/ceph/osd/{{ cluster }}-%i {% else %} ExecStartPre=-/usr/bin/{{ container_binary }} stop ceph-osd-%i {% endif %} ExecStartPre=-/usr/bin/{{ container_binary }} rm -f ceph-osd-%i -ExecStart={% if ceph_osd_numactl_opts != "" %} -numactl \ -{{ ceph_osd_numactl_opts }} \ -{% endif %} -/usr/bin/{{ container_binary }} run \ -{% if container_binary == 'podman' %} - -d --log-driver journald --conmon-pidfile /%t/%n-pid --cidfile /%t/%n-cid \ -{% endif %} - --pids-limit={{ 0 if container_binary == 'podman' else -1 }} \ - --security-opt label=disable \ - --rm \ - --net=host \ - --privileged=true \ - --pid=host \ - --ipc=host \ - --cpus={{ cpu_limit }} \ - {% if ceph_osd_docker_cpuset_cpus is defined -%} - --cpuset-cpus='{{ ceph_osd_docker_cpuset_cpus }}' \ - {% endif -%} - {% if ceph_osd_docker_cpuset_mems is defined -%} - --cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \ - {% endif -%} - -v /dev:/dev \ - -v /etc/localtime:/etc/localtime:ro \ - -v /var/lib/ceph/osd:/var/lib/ceph/osd:z \ - -v /var/lib/ceph/bootstrap-osd:/var/lib/ceph/bootstrap-osd:z \ - -v /etc/ceph:/etc/ceph:z \ - -v /var/run/ceph:/var/run/ceph:z \ - -v /var/run/udev/:/var/run/udev/ \ - -v /var/log/ceph:/var/log/ceph:z \ - {% if ansible_facts['distribution'] == 'Ubuntu' -%} - --security-opt apparmor:unconfined \ - {% endif -%} - {{ container_env_args }} \ - -e CLUSTER={{ cluster }} \ - -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ - -v /run/lvm/:/run/lvm/ \ - -e CEPH_DAEMON=OSD_CEPH_VOLUME_ACTIVATE \ - -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -e OSD_ID=%i \ - --name=ceph-osd-%i \ - {{ ceph_osd_docker_extra_env }} \ - {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} +ExecStart=/bin/bash /var/lib/ceph/osd/{{ cluster }}-%i/run %t %n {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" {% else %} diff --git a/roles/ceph-osd/templates/systemd-run.j2 b/roles/ceph-osd/templates/systemd-run.j2 new file mode 100644 index 000000000..b18631a23 --- /dev/null +++ b/roles/ceph-osd/templates/systemd-run.j2 @@ -0,0 +1,71 @@ +#!/bin/sh +{% set cpu_limit = ansible_facts['processor_vcpus']|int if ceph_osd_docker_cpu_limit|int > ansible_facts['processor_vcpus']|int else ceph_osd_docker_cpu_limit|int %} +OSD_ID={{ item }} +T=$1 +N=$2 +CEPH_VOLUME_CMD="/usr/bin/{{ container_binary }} run --rm --net=host --privileged=true --pid=host --ipc=host -v /dev:/dev -v /etc/localtime:/etc/localtime:ro -v /var/lib/ceph/osd/{{ cluster }}-${OSD_ID}:/var/lib/ceph/osd/{{ cluster }}-${OSD_ID}:z -v /etc/ceph:/etc/ceph:z -v /var/run/ceph:/var/run/ceph:z -v /var/run/udev/:/var/run/udev/ -v /var/log/ceph:/var/log/ceph:z -v /run/lvm/:/run/lvm/ --entrypoint=ceph-volume {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}" +PYTHON=python3 +CEPH_VOLUME_LIST_JSON="$($CEPH_VOLUME_CMD lvm list --format json)" + +# Find the OSD FSID from the OSD ID +OSD_FSID="$(echo "$CEPH_VOLUME_LIST_JSON" | $PYTHON -c "import sys, json; print(json.load(sys.stdin)['$OSD_ID'][0]['tags']['ceph.osd_fsid'])")" + +# Find the OSD type +OSD_TYPE="$(echo "$CEPH_VOLUME_LIST_JSON" | $PYTHON -c "import sys, json; print(json.load(sys.stdin)['$OSD_ID'][0]['type'])")" + +# Discover the objectstore +if [[ "data journal" =~ $OSD_TYPE ]]; then + OSD_OBJECTSTORE=(--filestore) +elif [[ "block wal db" =~ $OSD_TYPE ]]; then + OSD_OBJECTSTORE=(--bluestore) +else + log "Unable to discover osd objectstore for OSD type: $OSD_TYPE" + exit 1 +fi + +# activate +$CEPH_VOLUME_CMD lvm activate --no-systemd ${OSD_OBJECTSTORE[@]} ${OSD_ID} ${OSD_FSID} + +# start ceph-osd +{% if ceph_osd_numactl_opts != "" %} +numactl \ +{{ ceph_osd_numactl_opts }} \ +{% endif %} +/usr/bin/{{ container_binary }} run \ +{% if container_binary == 'podman' %} +-d --log-driver journald --conmon-pidfile /${T}/${N}-pid --cidfile /${T}/${N}-cid \ +{% endif %} +--pids-limit={{ 0 if container_binary == 'podman' else -1 }} \ +--rm --net=host --privileged=true --pid=host \ +--ipc=host \ +{% if osd_objectstore == 'filestore' -%} +--memory={{ ceph_osd_docker_memory_limit }} \ +{% endif -%} +--cpus={{ cpu_limit }} \ +{% if ceph_osd_docker_cpuset_cpus is defined -%} +--cpuset-cpus='{{ ceph_osd_docker_cpuset_cpus }}' \ +{% endif -%} +{% if ceph_osd_docker_cpuset_mems is defined -%} +--cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \ +{% endif -%} +-v /dev:/dev \ +-v /etc/localtime:/etc/localtime:ro \ +-v /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z \ +-v /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z \ +-v /etc/ceph:/etc/ceph:z \ +-v /var/run/ceph:/var/run/ceph:z \ +-v /var/run/udev/:/var/run/udev/ \ +-v /var/log/ceph:/var/log/ceph:z \ +{% if ansible_facts['distribution'] == 'Ubuntu' -%} +--security-opt apparmor:unconfined \ +{% endif -%} +{{ container_env_args }} \ +-e CLUSTER={{ cluster }} \ +-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ +-v /run/lvm/:/run/lvm/ \ +-e OSD_ID=${OSD_ID} \ +--name=ceph-osd-${OSD_ID} \ +--entrypoint=/usr/bin/ceph-osd \ +{{ ceph_osd_docker_extra_env }} \ +{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ +-f -i ${OSD_ID} \ No newline at end of file