Collocated mgr with mon fails to start on RHEL 8.7

With podman version podman-3:4.2.0-4.module+el8.7.0+17064+3b31f55c and
later, when mgr fails to start if mon is already running.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=2169767

Signed-off-by: Teoman ONAY <tonay@ibm.com>
(cherry picked from commit 637ca81c9c)
(cherry picked from commit be184e440f)

roles/ceph-crash/templates/ceph-crash.service.j2

@@ -20,7 +20,10 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-crash-%i \
 {% endif %}
 --pids-limit={{ 0 if container_binary == 'podman' else -1 }} \
 --net=host \
--v /var/lib/ceph:/var/lib/ceph:z \
+{% if cluster != 'ceph' %}
+-e CEPH_ARGS="--cluster {{ cluster }}" \
+{% endif %}
+-v /var/lib/ceph/crash:/var/lib/ceph/crash:z \
 -v /etc/localtime:/etc/localtime:ro \
 -v /etc/ceph:/etc/ceph:z \
 --entrypoint=/usr/bin/ceph-crash {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}

roles/ceph-mds/templates/ceph-mds.service.j2

@@ -27,7 +27,8 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --pids-limit={{ 0 if container_binary == 'podman' else -1 }} \
   --memory={{ ceph_mds_docker_memory_limit }} \
   --cpus={{ cpu_limit }} \
-  -v /var/lib/ceph:/var/lib/ceph:z \
+  -v /var/lib/ceph/mds:/var/lib/ceph/mds:z \
+  -v /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z \
   -v /etc/ceph:/etc/ceph:z \
   -v /var/run/ceph:/var/run/ceph:z \
   -v /etc/localtime:/etc/localtime:ro \

roles/ceph-mgr/templates/ceph-mgr.service.j2

@@ -26,7 +26,8 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --pids-limit={{ 0 if container_binary == 'podman' else -1 }} \
   --memory={{ ceph_mgr_docker_memory_limit }} \
   --cpus={{ ceph_mgr_docker_cpu_limit }} \
-  -v /var/lib/ceph:/var/lib/ceph:z,rshared \
+  -v /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared \
+  -v /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z \
   -v /etc/ceph:/etc/ceph:z \
   -v /var/run/ceph:/var/run/ceph:z \
   -v /etc/localtime:/etc/localtime:ro \

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -28,11 +28,11 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
   --memory={{ ceph_mon_docker_memory_limit }} \
   --cpus={{ ceph_mon_docker_cpu_limit }} \
   --security-opt label=disable \
-  -v /var/lib/ceph:/var/lib/ceph:rshared \
+  -v /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared \
-  -v /etc/ceph:/etc/ceph \
+  -v /etc/ceph:/etc/ceph:z \
-  -v /var/run/ceph:/var/run/ceph \
+  -v /var/run/ceph:/var/run/ceph:z \
-  -v /etc/localtime:/etc/localtime \
+  -v /etc/localtime:/etc/localtime:ro \
-  -v /var/log/ceph:/var/log/ceph \
+  -v /var/log/ceph:/var/log/ceph:z \
 {% if ansible_facts['os_family'] == 'RedHat' -%}
   -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
 {% endif -%}

roles/ceph-nfs/templates/ceph-nfs.service.j2

@@ -23,7 +23,7 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   -d --log-driver journald --conmon-pidfile /%t/%n-pid --cidfile /%t/%n-cid \
 {% endif %}
   --pids-limit={{ 0 if container_binary == 'podman' else -1 }} \
-  -v /var/lib/ceph:/var/lib/ceph:z \
+  -v /var/lib/ceph/bootstrap-rgw/:/var/lib/ceph/bootstrap-rgw:z \
   -v /etc/ceph:/etc/ceph:z \
   -v /var/lib/nfs/ganesha:/var/lib/nfs/ganesha:z \
   -v /etc/ganesha:/etc/ganesha:z \

roles/ceph-osd/templates/ceph-osd.service.j2

@@ -47,7 +47,8 @@ numactl \
   {% endif -%}
   -v /dev:/dev \
   -v /etc/localtime:/etc/localtime:ro \
-  -v /var/lib/ceph:/var/lib/ceph:z \
+  -v /var/lib/ceph/osd:/var/lib/ceph/osd:z \
+  -v /var/lib/ceph/bootstrap-osd:/var/lib/ceph/bootstrap-osd:z \
   -v /etc/ceph:/etc/ceph:z \
   -v /var/run/ceph:/var/run/ceph:z \
   -v /var/run/udev/:/var/run/udev/ \

roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2

@@ -26,7 +26,7 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --pids-limit={{ 0 if container_binary == 'podman' else -1 }} \
   --memory={{ ceph_rbd_mirror_docker_memory_limit }} \
   --cpus={{ ceph_rbd_mirror_docker_cpu_limit }} \
-  -v /var/lib/ceph:/var/lib/ceph:z \
+  -v /var/lib/ceph/bootstrap-rbd-mirror:/var/lib/ceph/bootstrap-rbd-mirror:Z \
   -v /etc/ceph:/etc/ceph:z \
   -v /var/run/ceph:/var/run/ceph:z \
   -v /etc/localtime:/etc/localtime:ro \

roles/ceph-rgw/templates/ceph-radosgw.service.j2

@@ -34,11 +34,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   {% if ceph_rgw_docker_cpuset_mems is defined -%}
   --cpuset-mems="{{ ceph_rgw_docker_cpuset_mems }}" \
   {% endif -%}
-  -v /var/lib/ceph:/var/lib/ceph \
+  -v /var/lib/ceph/radosgw:/var/lib/ceph/radosgw:z \
-  -v /etc/ceph:/etc/ceph \
+  -v /var/lib/ceph/bootstrap-rgw:/var/lib/ceph/bootstrap-rgw:z \
-  -v /var/run/ceph:/var/run/ceph \
+  -v /etc/ceph:/etc/ceph:z \
-  -v /etc/localtime:/etc/localtime \
+  -v /var/run/ceph:/var/run/ceph:z \
-  -v /var/log/ceph:/var/log/ceph \
+  -v /etc/localtime:/etc/localtime:ro \
+  -v /var/log/ceph:/var/log/ceph:z \
   {% if ansible_facts['os_family'] == 'RedHat' -%}
   -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
   {% endif -%}