ceph-osd: Add ulimit nofile on container start

On containerized deployment, the OSD entrypoint runs some ceph-volume
commands (lvm/simple scan and/or activate) which perform badly without
the ulimit option.
This option was added for all previous ceph-volume commands but not on
the ceph-osd container startup.
Also updating hard limit value to 4096 to reflect default baremetal
value.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1744390

Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>
(cherry picked from commit 9a4ac46d19)

library/ceph_volume.py

@@ -183,7 +183,7 @@ def container_exec(binary, container_image):
     Build the docker CLI to run a command inside a container
     '''
     command_exec = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',
-                    '--ulimit', 'nofile=1024:1024',
+                    '--ulimit', 'nofile=1024:4096',
                     '-v', '/run/lock/lvm:/run/lock/lvm:z',
                     '-v', '/var/run/udev/:/var/run/udev/:z',
                     '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',

library/test_ceph_volume.py

@@ -40,7 +40,7 @@ class TestCephVolumeModule(object):
         fake_binary = "ceph-volume"
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -57,7 +57,7 @@ class TestCephVolumeModule(object):
         fake_module.params = {'data': '/dev/sda'}
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -126,7 +126,7 @@ class TestCephVolumeModule(object):
         fake_module.params = {'cluster': 'ceph', 'data': '/dev/sda'}
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -154,7 +154,7 @@ class TestCephVolumeModule(object):
         fake_action = "create"
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -203,7 +203,7 @@ class TestCephVolumeModule(object):
         fake_action = "prepare"
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -253,7 +253,7 @@ class TestCephVolumeModule(object):
 
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501

roles/ceph-osd/tasks/scenarios/collocated.yml

@@ -5,7 +5,7 @@
 - name: prepare ceph containerized osd disk collocated
   shell: |
     docker run --net=host \
-    --ulimit nofile=1024:1024 \
+    --ulimit nofile=1024:4096 \
     --pid=host \
     --privileged=true \
     --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
@@ -31,7 +31,7 @@
 - name: automatic prepare ceph containerized osd disk collocated
   shell: |
     docker run --net=host \
-    --ulimit nofile=1024:1024 \
+    --ulimit nofile=1024:4096 \
     --pid=host \
     --privileged=true \
     --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.split('/')[-1] }} \

roles/ceph-osd/tasks/scenarios/non-collocated.yml

@@ -5,7 +5,7 @@
 - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated
   shell: |
     docker run --net=host \
-    --ulimit nofile=1024:1024 \
+    --ulimit nofile=1024:4096 \
     --pid=host \
     --privileged=true \
     --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
@@ -33,7 +33,7 @@
 - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated with a dedicated device for db
   shell: |
     docker run --net=host \
-    --ulimit nofile=1024:1024 \
+    --ulimit nofile=1024:4096 \
     --pid=host \
     --privileged=true \
     --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
@@ -62,7 +62,7 @@
 - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated with a dedicated device for db and wal
   shell: |
     docker run --net=host \
-    --ulimit nofile=1024:1024 \
+    --ulimit nofile=1024:4096 \
     --pid=host \
     --privileged=true \
     --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \

roles/ceph-osd/tasks/start_osds.yml

@@ -33,13 +33,13 @@
 
       - name: activate containerized osd(s)
         shell: |
-          DOCKER_ENV=$(docker run --rm --net=host --ulimit nofile=1024:1024 \
+          DOCKER_ENV=$(docker run --rm --net=host --ulimit nofile=1024:4096 \
           --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph:z \
           -e CLUSTER={{ cluster }} -e OSD_DEVICE={{ item }} \
           {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
           disk_list)
           docker run --rm --net=host \
-          --ulimit nofile=1024:1024 \
+          --ulimit nofile=1024:4096 \
           --ipc=host --pid=host --privileged=true \
           -v /etc/ceph:/etc/ceph:z \
           -v /var/lib/ceph/:/var/lib/ceph/:z \
@@ -80,7 +80,7 @@
 
 - name: set_fact docker_exec_start_osd
   set_fact:
-    docker_exec_start_osd: "{{ 'docker run --rm --ulimit nofile=1024:1024 --privileged=true -v /run/lvm/lvmetad.socket:/run/lvm/lvmetad.socket -v /var/run/udev/:/var/run/udev/:z -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}"
+    docker_exec_start_osd: "{{ 'docker run --rm --ulimit nofile=1024:4096 --privileged=true -v /run/lvm/lvmetad.socket:/run/lvm/lvmetad.socket -v /var/run/udev/:/var/run/udev/:z -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}"
   when: osd_scenario == 'lvm'
 
 - name: collect osd ids

roles/ceph-osd/templates/ceph-osd-run.sh.j2

@@ -14,9 +14,9 @@ DOCKER_ENV=""
 #############
 function id_to_device () {
 {% if dmcrypt | bool %}
-  docker run --rm --net=host --ulimit nofile=1024:1024 --ipc=host --pid=host --privileged=true -v /etc/ceph:/etc/ceph:z -v /var/lib/ceph/:/var/lib/ceph/:z -v /dev:/dev -v /etc/localtime:/etc/localtime:ro -e DEBUG=verbose -e CLUSTER={{ cluster }} {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} osd_ceph_disk_dmcrypt_data_map
+  docker run --rm --net=host --ulimit nofile=1024:4096 --ipc=host --pid=host --privileged=true -v /etc/ceph:/etc/ceph:z -v /var/lib/ceph/:/var/lib/ceph/:z -v /dev:/dev -v /etc/localtime:/etc/localtime:ro -e DEBUG=verbose -e CLUSTER={{ cluster }} {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} osd_ceph_disk_dmcrypt_data_map
 {% endif %}
-  DATA_PART=$(docker run --rm --ulimit nofile=1024:1024 --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph:z --entrypoint ceph-disk {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} list | grep ", osd\.${1}," | awk '{ print $1 }')
+  DATA_PART=$(docker run --rm --ulimit nofile=1024:4096 --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph:z --entrypoint ceph-disk {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} list | grep ", osd\.${1}," | awk '{ print $1 }')
   if [[ "${DATA_PART}" =~ ^/dev/(cciss|nvme) ]]; then
     OSD_DEVICE=${DATA_PART:0:-2}
   else
@@ -94,6 +94,7 @@ numactl \
   --privileged=true \
   --pid=host \
   --ipc=host \
+  --ulimit nofile=1024:4096 \
   {% if osd_objectstore == 'filestore' -%}
   --memory={{ ceph_osd_docker_memory_limit }} \
   {% endif -%}