From 9758e3c51303faf89d5c489f04539e278c49477e Mon Sep 17 00:00:00 2001 From: Dimitri Savineau Date: Tue, 29 Jun 2021 13:24:29 -0400 Subject: [PATCH] container: set tcmalloc value by default All ceph daemons need to have the TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES environment variable set to 128MB by default in container setup. Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1970913 Signed-off-by: Dimitri Savineau --- group_vars/all.yml.sample | 2 +- group_vars/rhcs.yml.sample | 2 +- .../tasks/configure_memory_allocator.yml | 16 ++++++++++++++-- roles/ceph-defaults/defaults/main.yml | 2 +- roles/ceph-mds/templates/ceph-mds.service.j2 | 1 + roles/ceph-mgr/templates/ceph-mgr.service.j2 | 1 + roles/ceph-mon/templates/ceph-mon.service.j2 | 1 + roles/ceph-osd/templates/ceph-osd.service.j2 | 2 -- .../templates/ceph-rbd-mirror.service.j2 | 1 + roles/ceph-rgw/templates/ceph-radosgw.service.j2 | 1 + site.yml.sample | 2 ++ 11 files changed, 24 insertions(+), 7 deletions(-) diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample index 88417ba58..d3ce14ca9 100644 --- a/group_vars/all.yml.sample +++ b/group_vars/all.yml.sample @@ -542,7 +542,7 @@ dummy: # For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES # Set this to a byte value (e.g. 134217728) # A value of 0 will leave the package default. -#ceph_tcmalloc_max_total_thread_cache: 0 +#ceph_tcmalloc_max_total_thread_cache: 134217728 ########## diff --git a/group_vars/rhcs.yml.sample b/group_vars/rhcs.yml.sample index 8d0e0e317..8f10e16db 100644 --- a/group_vars/rhcs.yml.sample +++ b/group_vars/rhcs.yml.sample @@ -542,7 +542,7 @@ ceph_iscsi_config_dev: false # For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES # Set this to a byte value (e.g. 134217728) # A value of 0 will leave the package default. -#ceph_tcmalloc_max_total_thread_cache: 0 +#ceph_tcmalloc_max_total_thread_cache: 134217728 ########## diff --git a/roles/ceph-common/tasks/configure_memory_allocator.yml b/roles/ceph-common/tasks/configure_memory_allocator.yml index 115ecf59e..2e81b629f 100644 --- a/roles/ceph-common/tasks/configure_memory_allocator.yml +++ b/roles/ceph-common/tasks/configure_memory_allocator.yml @@ -9,7 +9,13 @@ when: - ansible_facts['os_family'] == 'Debian' - etc_default_ceph.stat.exists - notify: restart ceph osds + notify: + - restart ceph mons + - restart ceph mgrs + - restart ceph osds + - restart ceph mdss + - restart ceph rgws + - restart ceph rbdmirrors - name: configure TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for redhat lineinfile: @@ -19,4 +25,10 @@ regexp: "^TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=" line: "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }}" when: ansible_facts['os_family'] == 'RedHat' - notify: restart ceph osds + notify: + - restart ceph mons + - restart ceph mgrs + - restart ceph osds + - restart ceph mdss + - restart ceph rgws + - restart ceph rbdmirrors diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml index 3350527ee..8c25c5dcc 100644 --- a/roles/ceph-defaults/defaults/main.yml +++ b/roles/ceph-defaults/defaults/main.yml @@ -534,7 +534,7 @@ os_tuning_params: # For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES # Set this to a byte value (e.g. 134217728) # A value of 0 will leave the package default. -ceph_tcmalloc_max_total_thread_cache: 0 +ceph_tcmalloc_max_total_thread_cache: 134217728 ########## diff --git a/roles/ceph-mds/templates/ceph-mds.service.j2 b/roles/ceph-mds/templates/ceph-mds.service.j2 index be5d000fd..c681146cb 100644 --- a/roles/ceph-mds/templates/ceph-mds.service.j2 +++ b/roles/ceph-mds/templates/ceph-mds.service.j2 @@ -32,6 +32,7 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ -e CLUSTER={{ cluster }} \ -e CEPH_DAEMON=MDS \ -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ + -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ {{ ceph_mds_docker_extra_env }} \ --name=ceph-mds-{{ ansible_facts['hostname'] }} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} diff --git a/roles/ceph-mgr/templates/ceph-mgr.service.j2 b/roles/ceph-mgr/templates/ceph-mgr.service.j2 index c7f7f42a7..56c49c4a6 100644 --- a/roles/ceph-mgr/templates/ceph-mgr.service.j2 +++ b/roles/ceph-mgr/templates/ceph-mgr.service.j2 @@ -31,6 +31,7 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ -e CLUSTER={{ cluster }} \ -e CEPH_DAEMON=MGR \ -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ + -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ {{ ceph_mgr_docker_extra_env }} \ --name=ceph-mgr-{{ ansible_facts['hostname'] }} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} diff --git a/roles/ceph-mon/templates/ceph-mon.service.j2 b/roles/ceph-mon/templates/ceph-mon.service.j2 index 32bc8c16b..741294dc4 100644 --- a/roles/ceph-mon/templates/ceph-mon.service.j2 +++ b/roles/ceph-mon/templates/ceph-mon.service.j2 @@ -46,6 +46,7 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \ -e CEPH_PUBLIC_NETWORK={{ public_network | regex_replace(' ', '') }} \ -e CEPH_DAEMON=MON \ -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ + -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ {{ ceph_mon_docker_extra_env }} \ {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} {% if container_binary == 'podman' %} diff --git a/roles/ceph-osd/templates/ceph-osd.service.j2 b/roles/ceph-osd/templates/ceph-osd.service.j2 index 8d06dce34..fb179dde5 100644 --- a/roles/ceph-osd/templates/ceph-osd.service.j2 +++ b/roles/ceph-osd/templates/ceph-osd.service.j2 @@ -54,9 +54,7 @@ numactl \ {% endif -%} {{ container_env_args }} \ -e CLUSTER={{ cluster }} \ - {% if (ceph_tcmalloc_max_total_thread_cache | int) > 0 -%} -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ - {% endif -%} -v /run/lvm/:/run/lvm/ \ -e CEPH_DAEMON=OSD_CEPH_VOLUME_ACTIVATE \ -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ diff --git a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 index f2ad5c186..48c0b3d17 100644 --- a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 +++ b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 @@ -31,6 +31,7 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ -e CLUSTER={{ cluster }} \ -e CEPH_DAEMON=RBD_MIRROR \ -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ + -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ --name=ceph-rbd-mirror-{{ ansible_facts['hostname'] }} \ {{ ceph_rbd_mirror_docker_extra_env }} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} diff --git a/roles/ceph-rgw/templates/ceph-radosgw.service.j2 b/roles/ceph-rgw/templates/ceph-radosgw.service.j2 index 9aec3bc5a..7c007e5bd 100644 --- a/roles/ceph-rgw/templates/ceph-radosgw.service.j2 +++ b/roles/ceph-rgw/templates/ceph-radosgw.service.j2 @@ -45,6 +45,7 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ -e CLUSTER={{ cluster }} \ -e RGW_NAME={{ ansible_facts['hostname'] }}.${INST_NAME} \ -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ + -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ --name=ceph-rgw-{{ ansible_facts['hostname'] }}-${INST_NAME} \ {{ ceph_rgw_docker_extra_env }} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} diff --git a/site.yml.sample b/site.yml.sample index e748e4c5f..8b09cfb03 100644 --- a/site.yml.sample +++ b/site.yml.sample @@ -68,6 +68,8 @@ - import_role: name: ceph-facts + - import_role: + name: ceph-handler - import_role: name: ceph-validate - import_role: