mirror of https://github.com/ceph/ceph-ansible.git
container: cleanup container systemd units
* Make common params of container args in a var to avoid duplication
* The /var/lib/ceph/crash mount was missing after 637ca81c9c
* Add CEPH_USE_RANDOM_NONCE as it's needed when running inside container (can be removed for squid later)
* Add NODE_NAME as some part of ceph code relies on this var
* add default logging opts for
Signed-off-by: Seena Fallah <seenafallah@gmail.com>
pull/7484/head
parent
c6d3bb5e92
commit
84e10bfd03
|
@ -534,7 +534,24 @@ dummy:
|
|||
#containerized_deployment: false
|
||||
#container_binary:
|
||||
#timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}"
|
||||
|
||||
#ceph_common_container_params:
|
||||
# envs:
|
||||
# NODE_NAME: "{{ ansible_facts['hostname'] }}"
|
||||
# CEPH_USE_RANDOM_NONCE: "1"
|
||||
# CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
|
||||
# TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
|
||||
# args:
|
||||
# - --setuser=ceph
|
||||
# - --setgroup=ceph
|
||||
# - --default-log-to-file=false
|
||||
# - --default-log-to-stderr=true
|
||||
# - --default-log-stderr-prefix="debug "
|
||||
# volumes:
|
||||
# - /var/lib/ceph/crash:/var/lib/ceph/crash:z
|
||||
# - /var/run/ceph:/var/run/ceph:z
|
||||
# - /var/log/ceph:/var/log/ceph:z
|
||||
# - /etc/ceph:/etc/ceph:z
|
||||
# - /etc/localtime:/etc/localtime:ro
|
||||
|
||||
# this is only here for usage with the rolling_update.yml playbook
|
||||
# do not ever change this here
|
||||
|
|
|
@ -13,6 +13,7 @@
|
|||
- /var/lib/ceph/osd
|
||||
- /var/lib/ceph/mds
|
||||
- /var/lib/ceph/tmp
|
||||
- /var/lib/ceph/crash
|
||||
- /var/lib/ceph/radosgw
|
||||
- /var/lib/ceph/bootstrap-rgw
|
||||
- /var/lib/ceph/bootstrap-mgr
|
||||
|
|
|
@ -24,9 +24,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-crash-%i \
|
|||
{% if cluster != 'ceph' %}
|
||||
-e CEPH_ARGS="--cluster {{ cluster }}" \
|
||||
{% endif %}
|
||||
-v /var/lib/ceph/crash:/var/lib/ceph/crash:z \
|
||||
-v /etc/localtime:/etc/localtime:ro \
|
||||
-v /etc/ceph:/etc/ceph:z \
|
||||
{% for v in ceph_common_container_params['volumes'] %}
|
||||
-v {{ v }} \
|
||||
{% endfor %}
|
||||
{% for k, v in ceph_common_container_params['envs'].items() %}
|
||||
-e {{ k }}={{ v }} \
|
||||
{% endfor %}
|
||||
--entrypoint=/usr/bin/ceph-crash {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
|
||||
{% if container_binary == 'podman' %}
|
||||
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
|
||||
|
|
|
@ -526,7 +526,22 @@ ceph_client_docker_registry: "{{ ceph_docker_registry }}"
|
|||
containerized_deployment: false
|
||||
container_binary:
|
||||
timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}"
|
||||
|
||||
ceph_common_container_params:
|
||||
envs:
|
||||
NODE_NAME: "{{ ansible_facts['hostname'] }}"
|
||||
CEPH_USE_RANDOM_NONCE: "1"
|
||||
CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
|
||||
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
|
||||
args:
|
||||
- --default-log-to-file=false
|
||||
- --default-log-to-stderr=true
|
||||
- --default-log-stderr-prefix="debug "
|
||||
volumes:
|
||||
- /var/lib/ceph/crash:/var/lib/ceph/crash:z
|
||||
- /var/run/ceph:/var/run/ceph:z
|
||||
- /var/log/ceph:/var/log/ceph:z
|
||||
- /etc/ceph:/etc/ceph:z
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
|
||||
# this is only here for usage with the rolling_update.yml playbook
|
||||
# do not ever change this here
|
||||
|
|
|
@ -197,7 +197,7 @@
|
|||
block:
|
||||
- &read-osd-pool-default-crush-rule
|
||||
name: Read osd pool default crush rule
|
||||
ansible.builtin.command: grep 'osd pool default crush rule' /etc/ceph/{{ cluster }}.conf
|
||||
ansible.builtin.command: grep 'osd_pool_default_crush_rule' /etc/ceph/{{ cluster }}.conf
|
||||
register: crush_rule_variable
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
|
|
@ -30,16 +30,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
|
|||
--cpus={{ cpu_limit }} \
|
||||
-v /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z \
|
||||
-v /var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:/var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:z \
|
||||
-v /etc/ceph:/etc/ceph:z \
|
||||
-v /var/run/ceph:/var/run/ceph:z \
|
||||
-v /etc/localtime:/etc/localtime:ro \
|
||||
-v /var/log/ceph:/var/log/ceph:z \
|
||||
-e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
|
||||
-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
|
||||
{% for v in ceph_common_container_params['volumes'] %}
|
||||
-v {{ v }} \
|
||||
{% endfor %}
|
||||
{% for k, v in ceph_common_container_params['envs'].items() %}
|
||||
-e {{ k }}={{ v }} \
|
||||
{% endfor %}
|
||||
{{ ceph_mds_docker_extra_env }} \
|
||||
--name=ceph-mds-{{ ansible_facts['hostname'] }} \
|
||||
--entrypoint=/usr/bin/ceph-mds \
|
||||
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
|
||||
{% for arg in ceph_common_container_params['args'] %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
-f -i {{ ansible_facts['hostname'] }}
|
||||
{% if container_binary == 'podman' %}
|
||||
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
|
||||
|
|
|
@ -27,21 +27,22 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
|
|||
--security-opt label=disable \
|
||||
--memory={{ ceph_mgr_docker_memory_limit }} \
|
||||
--cpus={{ ceph_mgr_docker_cpu_limit }} \
|
||||
{% for v in ceph_common_container_params['volumes'] %}
|
||||
-v {{ v }} \
|
||||
{% endfor %}
|
||||
-v /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared \
|
||||
-v /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z \
|
||||
-v /etc/ceph:/etc/ceph:z \
|
||||
-v /var/run/ceph:/var/run/ceph:z \
|
||||
-v /etc/localtime:/etc/localtime:ro \
|
||||
-v /var/log/ceph:/var/log/ceph:z \
|
||||
-e CLUSTER={{ cluster }} \
|
||||
-e CEPH_DAEMON=MGR \
|
||||
-e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
|
||||
-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
|
||||
{% for k, v in ceph_common_container_params['envs'].items() %}
|
||||
-e {{ k }}={{ v }} \
|
||||
{% endfor %}
|
||||
{{ ceph_mgr_docker_extra_env }} \
|
||||
--name=ceph-mgr-{{ ansible_facts['hostname'] }} \
|
||||
--entrypoint=/usr/bin/ceph-mgr \
|
||||
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
|
||||
-f --default-log-to-file=false --default-log-to-stderr=true \
|
||||
{% for arg in ceph_common_container_params['args'] %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
-f \
|
||||
-i {{ ansible_facts['hostname'] }}
|
||||
{% if container_binary == 'podman' %}
|
||||
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
|
||||
|
|
|
@ -28,11 +28,10 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
|
|||
--memory={{ ceph_mon_docker_memory_limit }} \
|
||||
--cpus={{ ceph_mon_docker_cpu_limit }} \
|
||||
--security-opt label=disable \
|
||||
{% for v in ceph_common_container_params['volumes'] %}
|
||||
-v {{ v }} \
|
||||
{% endfor %}
|
||||
-v /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared \
|
||||
-v /etc/ceph:/etc/ceph:z \
|
||||
-v /var/run/ceph:/var/run/ceph:z \
|
||||
-v /etc/localtime:/etc/localtime:ro \
|
||||
-v /var/log/ceph:/var/log/ceph:z \
|
||||
{% if ansible_facts['os_family'] == 'RedHat' -%}
|
||||
-v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
|
||||
{% endif -%}
|
||||
|
@ -42,19 +41,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
|
|||
{% if mon_docker_net_host | bool -%}
|
||||
--net=host \
|
||||
{% endif -%}
|
||||
-e IP_VERSION={{ ip_version[-1:] }} \
|
||||
-e MON_IP={{ _current_monitor_address }} \
|
||||
-e CLUSTER={{ cluster }} \
|
||||
-e FSID={{ fsid }} \
|
||||
-e MON_PORT={{ ceph_mon_container_listen_port }} \
|
||||
-e CEPH_PUBLIC_NETWORK={{ public_network | regex_replace(' ', '') }} \
|
||||
-e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
|
||||
-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
|
||||
{% for k, v in ceph_common_container_params['envs'].items() %}
|
||||
-e {{ k }}={{ v }} \
|
||||
{% endfor %}
|
||||
{{ ceph_mon_docker_extra_env }} \
|
||||
--entrypoint=/usr/bin/ceph-mon \
|
||||
{{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
|
||||
-f --default-log-to-file=false --default-log-to-stderr=true \
|
||||
-i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name][0] }}
|
||||
{% for arg in ceph_common_container_params['args'] %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
-f \
|
||||
--default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true \
|
||||
-i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} \
|
||||
--public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name] | join(',') }}
|
||||
{% if container_binary == 'podman' %}
|
||||
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
|
||||
{% else %}
|
||||
|
|
|
@ -63,7 +63,7 @@
|
|||
community.general.ini_file:
|
||||
dest: "/etc/ceph/{{ cluster }}.conf"
|
||||
section: "global"
|
||||
option: "osd pool default crush rule"
|
||||
option: "osd_pool_default_crush_rule"
|
||||
value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}"
|
||||
mode: "0644"
|
||||
delegate_to: "{{ item }}"
|
||||
|
|
|
@ -43,23 +43,26 @@ numactl \
|
|||
{% if ceph_osd_docker_cpuset_mems is defined -%}
|
||||
--cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \
|
||||
{% endif -%}
|
||||
{% for v in ceph_common_container_params['volumes'] %}
|
||||
-v {{ v }} \
|
||||
{% endfor %}
|
||||
-v /dev:/dev \
|
||||
-v /etc/localtime:/etc/localtime:ro \
|
||||
-v /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z \
|
||||
-v /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z \
|
||||
-v /etc/ceph:/etc/ceph:z \
|
||||
-v /var/run/ceph:/var/run/ceph:z \
|
||||
-v /var/run/udev/:/var/run/udev/ \
|
||||
-v /var/log/ceph:/var/log/ceph:z \
|
||||
{% if ansible_facts['distribution'] == 'Ubuntu' -%}
|
||||
--security-opt apparmor:unconfined \
|
||||
{% endif -%}
|
||||
-e CLUSTER={{ cluster }} \
|
||||
-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
|
||||
{% for k, v in ceph_common_container_params['envs'].items() %}
|
||||
-e {{ k }}={{ v }} \
|
||||
{% endfor %}
|
||||
-v /run/lvm/:/run/lvm/ \
|
||||
-e OSD_ID=${OSD_ID} \
|
||||
--name=ceph-osd-${OSD_ID} \
|
||||
--entrypoint=/usr/bin/ceph-osd \
|
||||
{{ ceph_osd_docker_extra_env }} \
|
||||
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
|
||||
-f -i ${OSD_ID}
|
||||
{% for arg in ceph_common_container_params['args'] %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
-f -i ${OSD_ID}
|
||||
|
|
|
@ -34,11 +34,13 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
|
|||
{% if ceph_rgw_docker_cpuset_mems is defined -%}
|
||||
--cpuset-mems="{{ ceph_rgw_docker_cpuset_mems }}" \
|
||||
{% endif -%}
|
||||
{% for v in ceph_common_container_params['volumes'] %}
|
||||
-v {{ v }} \
|
||||
{% endfor %}
|
||||
-v /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:z \
|
||||
-v /etc/ceph:/etc/ceph \
|
||||
-v /var/run/ceph:/var/run/ceph \
|
||||
-v /etc/localtime:/etc/localtime \
|
||||
-v /var/log/ceph:/var/log/ceph \
|
||||
{% for k, v in ceph_common_container_params['envs'].items() %}
|
||||
-e {{ k }}={{ v }} \
|
||||
{% endfor %}
|
||||
{% if ansible_facts['os_family'] == 'RedHat' -%}
|
||||
-v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
|
||||
{% endif -%}
|
||||
|
@ -50,6 +52,9 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
|
|||
--entrypoint=/usr/bin/radosgw \
|
||||
{{ ceph_rgw_docker_extra_env }} \
|
||||
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
|
||||
{% for arg in ceph_common_container_params['args'] %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
-f -n client.rgw.{{ ansible_facts['hostname'] }}.${INST_NAME} -k /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}/keyring
|
||||
{% if container_binary == 'podman' %}
|
||||
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
|
||||
|
|
Loading…
Reference in New Issue