container: cleanup container systemd units

* Make common params of container args in a var to avoid duplication
* The /var/lib/ceph/crash mount was missing after 637ca81c9c
* Add CEPH_USE_RANDOM_NONCE as it's needed when running inside container (can be removed for squid later)
* Add NODE_NAME as some part of ceph code relies on this var
* add default logging opts for

Signed-off-by: Seena Fallah <seenafallah@gmail.com>
pull/7484/head
Seena Fallah 2024-02-18 03:41:41 +01:00 committed by Guillaume Abrioux
parent c6d3bb5e92
commit 84e10bfd03
11 changed files with 94 additions and 47 deletions

View File

@ -534,7 +534,24 @@ dummy:
#containerized_deployment: false
#container_binary:
#timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}"
#ceph_common_container_params:
# envs:
# NODE_NAME: "{{ ansible_facts['hostname'] }}"
# CEPH_USE_RANDOM_NONCE: "1"
# CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
# TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
# args:
# - --setuser=ceph
# - --setgroup=ceph
# - --default-log-to-file=false
# - --default-log-to-stderr=true
# - --default-log-stderr-prefix="debug "
# volumes:
# - /var/lib/ceph/crash:/var/lib/ceph/crash:z
# - /var/run/ceph:/var/run/ceph:z
# - /var/log/ceph:/var/log/ceph:z
# - /etc/ceph:/etc/ceph:z
# - /etc/localtime:/etc/localtime:ro
# this is only here for usage with the rolling_update.yml playbook
# do not ever change this here

View File

@ -13,6 +13,7 @@
- /var/lib/ceph/osd
- /var/lib/ceph/mds
- /var/lib/ceph/tmp
- /var/lib/ceph/crash
- /var/lib/ceph/radosgw
- /var/lib/ceph/bootstrap-rgw
- /var/lib/ceph/bootstrap-mgr

View File

@ -24,9 +24,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-crash-%i \
{% if cluster != 'ceph' %}
-e CEPH_ARGS="--cluster {{ cluster }}" \
{% endif %}
-v /var/lib/ceph/crash:/var/lib/ceph/crash:z \
-v /etc/localtime:/etc/localtime:ro \
-v /etc/ceph:/etc/ceph:z \
{% for v in ceph_common_container_params['volumes'] %}
-v {{ v }} \
{% endfor %}
{% for k, v in ceph_common_container_params['envs'].items() %}
-e {{ k }}={{ v }} \
{% endfor %}
--entrypoint=/usr/bin/ceph-crash {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
{% if container_binary == 'podman' %}
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

View File

@ -526,7 +526,22 @@ ceph_client_docker_registry: "{{ ceph_docker_registry }}"
containerized_deployment: false
container_binary:
timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}"
ceph_common_container_params:
envs:
NODE_NAME: "{{ ansible_facts['hostname'] }}"
CEPH_USE_RANDOM_NONCE: "1"
CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
args:
- --default-log-to-file=false
- --default-log-to-stderr=true
- --default-log-stderr-prefix="debug "
volumes:
- /var/lib/ceph/crash:/var/lib/ceph/crash:z
- /var/run/ceph:/var/run/ceph:z
- /var/log/ceph:/var/log/ceph:z
- /etc/ceph:/etc/ceph:z
- /etc/localtime:/etc/localtime:ro
# this is only here for usage with the rolling_update.yml playbook
# do not ever change this here

View File

@ -197,7 +197,7 @@
block:
- &read-osd-pool-default-crush-rule
name: Read osd pool default crush rule
ansible.builtin.command: grep 'osd pool default crush rule' /etc/ceph/{{ cluster }}.conf
ansible.builtin.command: grep 'osd_pool_default_crush_rule' /etc/ceph/{{ cluster }}.conf
register: crush_rule_variable
changed_when: false
check_mode: false

View File

@ -30,16 +30,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
--cpus={{ cpu_limit }} \
-v /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z \
-v /var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:/var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:z \
-v /etc/ceph:/etc/ceph:z \
-v /var/run/ceph:/var/run/ceph:z \
-v /etc/localtime:/etc/localtime:ro \
-v /var/log/ceph:/var/log/ceph:z \
-e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
{% for v in ceph_common_container_params['volumes'] %}
-v {{ v }} \
{% endfor %}
{% for k, v in ceph_common_container_params['envs'].items() %}
-e {{ k }}={{ v }} \
{% endfor %}
{{ ceph_mds_docker_extra_env }} \
--name=ceph-mds-{{ ansible_facts['hostname'] }} \
--entrypoint=/usr/bin/ceph-mds \
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
{% for arg in ceph_common_container_params['args'] %}
{{ arg }} \
{% endfor %}
-f -i {{ ansible_facts['hostname'] }}
{% if container_binary == 'podman' %}
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

View File

@ -27,21 +27,22 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
--security-opt label=disable \
--memory={{ ceph_mgr_docker_memory_limit }} \
--cpus={{ ceph_mgr_docker_cpu_limit }} \
{% for v in ceph_common_container_params['volumes'] %}
-v {{ v }} \
{% endfor %}
-v /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared \
-v /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z \
-v /etc/ceph:/etc/ceph:z \
-v /var/run/ceph:/var/run/ceph:z \
-v /etc/localtime:/etc/localtime:ro \
-v /var/log/ceph:/var/log/ceph:z \
-e CLUSTER={{ cluster }} \
-e CEPH_DAEMON=MGR \
-e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
{% for k, v in ceph_common_container_params['envs'].items() %}
-e {{ k }}={{ v }} \
{% endfor %}
{{ ceph_mgr_docker_extra_env }} \
--name=ceph-mgr-{{ ansible_facts['hostname'] }} \
--entrypoint=/usr/bin/ceph-mgr \
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-f --default-log-to-file=false --default-log-to-stderr=true \
{% for arg in ceph_common_container_params['args'] %}
{{ arg }} \
{% endfor %}
-f \
-i {{ ansible_facts['hostname'] }}
{% if container_binary == 'podman' %}
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

View File

@ -28,11 +28,10 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
--memory={{ ceph_mon_docker_memory_limit }} \
--cpus={{ ceph_mon_docker_cpu_limit }} \
--security-opt label=disable \
{% for v in ceph_common_container_params['volumes'] %}
-v {{ v }} \
{% endfor %}
-v /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared \
-v /etc/ceph:/etc/ceph:z \
-v /var/run/ceph:/var/run/ceph:z \
-v /etc/localtime:/etc/localtime:ro \
-v /var/log/ceph:/var/log/ceph:z \
{% if ansible_facts['os_family'] == 'RedHat' -%}
-v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
{% endif -%}
@ -42,19 +41,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
{% if mon_docker_net_host | bool -%}
--net=host \
{% endif -%}
-e IP_VERSION={{ ip_version[-1:] }} \
-e MON_IP={{ _current_monitor_address }} \
-e CLUSTER={{ cluster }} \
-e FSID={{ fsid }} \
-e MON_PORT={{ ceph_mon_container_listen_port }} \
-e CEPH_PUBLIC_NETWORK={{ public_network | regex_replace(' ', '') }} \
-e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
{% for k, v in ceph_common_container_params['envs'].items() %}
-e {{ k }}={{ v }} \
{% endfor %}
{{ ceph_mon_docker_extra_env }} \
--entrypoint=/usr/bin/ceph-mon \
{{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-f --default-log-to-file=false --default-log-to-stderr=true \
-i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name][0] }}
{% for arg in ceph_common_container_params['args'] %}
{{ arg }} \
{% endfor %}
-f \
--default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true \
-i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} \
--public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name] | join(',') }}
{% if container_binary == 'podman' %}
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
{% else %}

View File

@ -63,7 +63,7 @@
community.general.ini_file:
dest: "/etc/ceph/{{ cluster }}.conf"
section: "global"
option: "osd pool default crush rule"
option: "osd_pool_default_crush_rule"
value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}"
mode: "0644"
delegate_to: "{{ item }}"

View File

@ -43,23 +43,26 @@ numactl \
{% if ceph_osd_docker_cpuset_mems is defined -%}
--cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \
{% endif -%}
{% for v in ceph_common_container_params['volumes'] %}
-v {{ v }} \
{% endfor %}
-v /dev:/dev \
-v /etc/localtime:/etc/localtime:ro \
-v /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z \
-v /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z \
-v /etc/ceph:/etc/ceph:z \
-v /var/run/ceph:/var/run/ceph:z \
-v /var/run/udev/:/var/run/udev/ \
-v /var/log/ceph:/var/log/ceph:z \
{% if ansible_facts['distribution'] == 'Ubuntu' -%}
--security-opt apparmor:unconfined \
{% endif -%}
-e CLUSTER={{ cluster }} \
-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
{% for k, v in ceph_common_container_params['envs'].items() %}
-e {{ k }}={{ v }} \
{% endfor %}
-v /run/lvm/:/run/lvm/ \
-e OSD_ID=${OSD_ID} \
--name=ceph-osd-${OSD_ID} \
--entrypoint=/usr/bin/ceph-osd \
{{ ceph_osd_docker_extra_env }} \
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-f -i ${OSD_ID}
{% for arg in ceph_common_container_params['args'] %}
{{ arg }} \
{% endfor %}
-f -i ${OSD_ID}

View File

@ -34,11 +34,13 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
{% if ceph_rgw_docker_cpuset_mems is defined -%}
--cpuset-mems="{{ ceph_rgw_docker_cpuset_mems }}" \
{% endif -%}
{% for v in ceph_common_container_params['volumes'] %}
-v {{ v }} \
{% endfor %}
-v /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:z \
-v /etc/ceph:/etc/ceph \
-v /var/run/ceph:/var/run/ceph \
-v /etc/localtime:/etc/localtime \
-v /var/log/ceph:/var/log/ceph \
{% for k, v in ceph_common_container_params['envs'].items() %}
-e {{ k }}={{ v }} \
{% endfor %}
{% if ansible_facts['os_family'] == 'RedHat' -%}
-v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
{% endif -%}
@ -50,6 +52,9 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
--entrypoint=/usr/bin/radosgw \
{{ ceph_rgw_docker_extra_env }} \
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
{% for arg in ceph_common_container_params['args'] %}
{{ arg }} \
{% endfor %}
-f -n client.rgw.{{ ansible_facts['hostname'] }}.${INST_NAME} -k /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}/keyring
{% if container_binary == 'podman' %}
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"