container: cleanup container systemd units

* Make common params of container args in a var to avoid duplication
* The /var/lib/ceph/crash mount was missing after 637ca81c9c
* Add CEPH_USE_RANDOM_NONCE as it's needed when running inside container (can be removed for squid later)
* Add NODE_NAME as some part of ceph code relies on this var
* add default logging opts for

Signed-off-by: Seena Fallah <seenafallah@gmail.com>

group_vars/all.yml.sample

@@ -534,7 +534,24 @@ dummy:
 #containerized_deployment: false
 #container_binary:
 #timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}"
-
+#ceph_common_container_params:
+#  envs:
+#    NODE_NAME: "{{ ansible_facts['hostname'] }}"
+#    CEPH_USE_RANDOM_NONCE: "1"
+#    CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
+#    TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
+#  args:
+#    - --setuser=ceph
+#    - --setgroup=ceph
+#    - --default-log-to-file=false
+#    - --default-log-to-stderr=true
+#    - --default-log-stderr-prefix="debug "
+#  volumes:
+#    - /var/lib/ceph/crash:/var/lib/ceph/crash:z
+#    - /var/run/ceph:/var/run/ceph:z
+#    - /var/log/ceph:/var/log/ceph:z
+#    - /etc/ceph:/etc/ceph:z
+#    - /etc/localtime:/etc/localtime:ro
 
 # this is only here for usage with the rolling_update.yml playbook
 # do not ever change this here

roles/ceph-config/tasks/create_ceph_initial_dirs.yml

@@ -13,6 +13,7 @@
     - /var/lib/ceph/osd
     - /var/lib/ceph/mds
     - /var/lib/ceph/tmp
+    - /var/lib/ceph/crash
     - /var/lib/ceph/radosgw
     - /var/lib/ceph/bootstrap-rgw
     - /var/lib/ceph/bootstrap-mgr

roles/ceph-crash/templates/ceph-crash.service.j2

@@ -24,9 +24,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-crash-%i \
 {% if cluster != 'ceph' %}
 -e CEPH_ARGS="--cluster {{ cluster }}" \
 {% endif %}
--v /var/lib/ceph/crash:/var/lib/ceph/crash:z \
+{% for v in ceph_common_container_params['volumes'] %}
--v /etc/localtime:/etc/localtime:ro \
+  -v {{ v }} \
--v /etc/ceph:/etc/ceph:z \
+{% endfor %}
+{% for k, v in ceph_common_container_params['envs'].items() %}
+  -e {{ k }}={{ v }} \
+{% endfor %}
 --entrypoint=/usr/bin/ceph-crash {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

roles/ceph-defaults/defaults/main.yml

@@ -526,7 +526,22 @@ ceph_client_docker_registry: "{{ ceph_docker_registry }}"
 containerized_deployment: false
 container_binary:
 timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}"
-
+ceph_common_container_params:
+  envs:
+    NODE_NAME: "{{ ansible_facts['hostname'] }}"
+    CEPH_USE_RANDOM_NONCE: "1"
+    CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
+    TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
+  args:
+    - --default-log-to-file=false
+    - --default-log-to-stderr=true
+    - --default-log-stderr-prefix="debug "
+  volumes:
+    - /var/lib/ceph/crash:/var/lib/ceph/crash:z
+    - /var/run/ceph:/var/run/ceph:z
+    - /var/log/ceph:/var/log/ceph:z
+    - /etc/ceph:/etc/ceph:z
+    - /etc/localtime:/etc/localtime:ro
 
 # this is only here for usage with the rolling_update.yml playbook
 # do not ever change this here

roles/ceph-facts/tasks/facts.yml

@@ -197,7 +197,7 @@
   block:
     - &read-osd-pool-default-crush-rule
       name: Read osd pool default crush rule
-      ansible.builtin.command: grep 'osd pool default crush rule' /etc/ceph/{{ cluster }}.conf
+      ansible.builtin.command: grep 'osd_pool_default_crush_rule' /etc/ceph/{{ cluster }}.conf
       register: crush_rule_variable
       changed_when: false
       check_mode: false

roles/ceph-mds/templates/ceph-mds.service.j2

@@ -30,16 +30,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --cpus={{ cpu_limit }} \
   -v /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z \
   -v /var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:/var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:z \
-  -v /etc/ceph:/etc/ceph:z \
+{% for v in ceph_common_container_params['volumes'] %}
-  -v /var/run/ceph:/var/run/ceph:z \
+  -v {{ v }} \
-  -v /etc/localtime:/etc/localtime:ro \
+{% endfor %}
-  -v /var/log/ceph:/var/log/ceph:z \
+{% for k, v in ceph_common_container_params['envs'].items() %}
-  -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
+  -e {{ k }}={{ v }} \
-  -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
+{% endfor %}
   {{ ceph_mds_docker_extra_env }} \
   --name=ceph-mds-{{ ansible_facts['hostname'] }} \
   --entrypoint=/usr/bin/ceph-mds \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
+{% for arg in ceph_common_container_params['args'] %}
+  {{ arg }} \
+{% endfor %}
   -f -i {{ ansible_facts['hostname'] }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

roles/ceph-mgr/templates/ceph-mgr.service.j2

@@ -27,21 +27,22 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --security-opt label=disable \
   --memory={{ ceph_mgr_docker_memory_limit }} \
   --cpus={{ ceph_mgr_docker_cpu_limit }} \
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
   -v /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared \
   -v /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z \
-  -v /etc/ceph:/etc/ceph:z \
+{% for k, v in ceph_common_container_params['envs'].items() %}
-  -v /var/run/ceph:/var/run/ceph:z \
+  -e {{ k }}={{ v }} \
-  -v /etc/localtime:/etc/localtime:ro \
+{% endfor %}
-  -v /var/log/ceph:/var/log/ceph:z \
-  -e CLUSTER={{ cluster }} \
-  -e CEPH_DAEMON=MGR \
-  -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-  -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
   {{ ceph_mgr_docker_extra_env }} \
   --name=ceph-mgr-{{ ansible_facts['hostname'] }} \
   --entrypoint=/usr/bin/ceph-mgr \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-  -f --default-log-to-file=false --default-log-to-stderr=true \
+{% for arg in ceph_common_container_params['args'] %}
+  {{ arg }} \
+{% endfor %}
+  -f \
   -i {{ ansible_facts['hostname'] }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -28,11 +28,10 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
   --memory={{ ceph_mon_docker_memory_limit }} \
   --cpus={{ ceph_mon_docker_cpu_limit }} \
   --security-opt label=disable \
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
   -v /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared \
-  -v /etc/ceph:/etc/ceph:z \
-  -v /var/run/ceph:/var/run/ceph:z \
-  -v /etc/localtime:/etc/localtime:ro \
-  -v /var/log/ceph:/var/log/ceph:z \
 {% if ansible_facts['os_family'] == 'RedHat' -%}
   -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
 {% endif -%}
@@ -42,19 +41,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
 {% if mon_docker_net_host | bool -%}
   --net=host \
 {% endif -%}
-  -e IP_VERSION={{ ip_version[-1:] }} \
+{% for k, v in ceph_common_container_params['envs'].items() %}
-  -e MON_IP={{ _current_monitor_address }} \
+  -e {{ k }}={{ v }} \
-  -e CLUSTER={{ cluster }} \
+{% endfor %}
-  -e FSID={{ fsid }} \
-  -e MON_PORT={{ ceph_mon_container_listen_port }} \
-  -e CEPH_PUBLIC_NETWORK={{ public_network | regex_replace(' ', '') }} \
-  -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-  -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
   {{ ceph_mon_docker_extra_env }} \
   --entrypoint=/usr/bin/ceph-mon \
   {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-  -f --default-log-to-file=false --default-log-to-stderr=true \
+{% for arg in ceph_common_container_params['args'] %}
-  -i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name][0] }}
+  {{ arg }} \
+{% endfor %}
+  -f \
+  --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true \
+  -i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} \
+  --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name] | join(',') }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
 {% else %}

roles/ceph-osd/tasks/crush_rules.yml

@@ -63,7 +63,7 @@
   community.general.ini_file:
     dest: "/etc/ceph/{{ cluster }}.conf"
     section: "global"
-    option: "osd pool default crush rule"
+    option: "osd_pool_default_crush_rule"
     value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}"
     mode: "0644"
   delegate_to: "{{ item }}"

roles/ceph-osd/templates/systemd-run.j2

@@ -43,23 +43,26 @@ numactl \
 {% if ceph_osd_docker_cpuset_mems is defined -%}
 --cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \
 {% endif -%}
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
 -v /dev:/dev \
--v /etc/localtime:/etc/localtime:ro \
 -v /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z \
 -v /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z \
--v /etc/ceph:/etc/ceph:z \
--v /var/run/ceph:/var/run/ceph:z \
 -v /var/run/udev/:/var/run/udev/ \
--v /var/log/ceph:/var/log/ceph:z \
 {% if ansible_facts['distribution'] == 'Ubuntu' -%}
 --security-opt apparmor:unconfined \
 {% endif -%}
--e CLUSTER={{ cluster }} \
+{% for k, v in ceph_common_container_params['envs'].items() %}
--e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
+-e {{ k }}={{ v }} \
+{% endfor %}
 -v /run/lvm/:/run/lvm/ \
 -e OSD_ID=${OSD_ID} \
 --name=ceph-osd-${OSD_ID} \
 --entrypoint=/usr/bin/ceph-osd \
 {{ ceph_osd_docker_extra_env }} \
 {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
--f -i ${OSD_ID}
+{% for arg in ceph_common_container_params['args'] %}
+  {{ arg }} \
+{% endfor %}
+-f -i ${OSD_ID}

roles/ceph-rgw/templates/ceph-radosgw.service.j2

@@ -34,11 +34,13 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   {% if ceph_rgw_docker_cpuset_mems is defined -%}
   --cpuset-mems="{{ ceph_rgw_docker_cpuset_mems }}" \
   {% endif -%}
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
   -v /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:z \
-  -v /etc/ceph:/etc/ceph \
+{% for k, v in ceph_common_container_params['envs'].items() %}
-  -v /var/run/ceph:/var/run/ceph \
+  -e {{ k }}={{ v }} \
-  -v /etc/localtime:/etc/localtime \
+{% endfor %}
-  -v /var/log/ceph:/var/log/ceph \
   {% if ansible_facts['os_family'] == 'RedHat' -%}
   -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
   {% endif -%}
@@ -50,6 +52,9 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --entrypoint=/usr/bin/radosgw \
   {{ ceph_rgw_docker_extra_env }} \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
+{% for arg in ceph_common_container_params['args'] %}
+  {{ arg }} \
+{% endfor %}
   -f -n client.rgw.{{ ansible_facts['hostname'] }}.${INST_NAME} -k /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}/keyring
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"