diff --git a/library/ceph_volume.py b/library/ceph_volume.py index 2600c20fe..3c4fb885b 100644 --- a/library/ceph_volume.py +++ b/library/ceph_volume.py @@ -183,7 +183,7 @@ def container_exec(binary, container_image): Build the docker CLI to run a command inside a container ''' command_exec = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', diff --git a/library/test_ceph_volume.py b/library/test_ceph_volume.py index 11eb9bb66..3f6d6508b 100644 --- a/library/test_ceph_volume.py +++ b/library/test_ceph_volume.py @@ -40,7 +40,7 @@ class TestCephVolumeModule(object): fake_binary = "ceph-volume" fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -57,7 +57,7 @@ class TestCephVolumeModule(object): fake_module.params = {'data': '/dev/sda'} fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -126,7 +126,7 @@ class TestCephVolumeModule(object): fake_module.params = {'cluster': 'ceph', 'data': '/dev/sda'} fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -154,7 +154,7 @@ class TestCephVolumeModule(object): fake_action = "create" fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -203,7 +203,7 @@ class TestCephVolumeModule(object): fake_action = "prepare" fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -253,7 +253,7 @@ class TestCephVolumeModule(object): fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 diff --git a/roles/ceph-osd/tasks/scenarios/collocated.yml b/roles/ceph-osd/tasks/scenarios/collocated.yml index 946877472..a075c547d 100644 --- a/roles/ceph-osd/tasks/scenarios/collocated.yml +++ b/roles/ceph-osd/tasks/scenarios/collocated.yml @@ -5,7 +5,7 @@ - name: prepare ceph containerized osd disk collocated shell: | docker run --net=host \ - --ulimit nofile=1024:1024 \ + --ulimit nofile=1024:4096 \ --pid=host \ --privileged=true \ --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \ @@ -31,7 +31,7 @@ - name: automatic prepare ceph containerized osd disk collocated shell: | docker run --net=host \ - --ulimit nofile=1024:1024 \ + --ulimit nofile=1024:4096 \ --pid=host \ --privileged=true \ --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.split('/')[-1] }} \ diff --git a/roles/ceph-osd/tasks/scenarios/non-collocated.yml b/roles/ceph-osd/tasks/scenarios/non-collocated.yml index f3fb858d5..ee6ed48bf 100644 --- a/roles/ceph-osd/tasks/scenarios/non-collocated.yml +++ b/roles/ceph-osd/tasks/scenarios/non-collocated.yml @@ -5,7 +5,7 @@ - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated shell: | docker run --net=host \ - --ulimit nofile=1024:1024 \ + --ulimit nofile=1024:4096 \ --pid=host \ --privileged=true \ --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \ @@ -33,7 +33,7 @@ - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated with a dedicated device for db shell: | docker run --net=host \ - --ulimit nofile=1024:1024 \ + --ulimit nofile=1024:4096 \ --pid=host \ --privileged=true \ --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \ @@ -62,7 +62,7 @@ - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated with a dedicated device for db and wal shell: | docker run --net=host \ - --ulimit nofile=1024:1024 \ + --ulimit nofile=1024:4096 \ --pid=host \ --privileged=true \ --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \ diff --git a/roles/ceph-osd/tasks/start_osds.yml b/roles/ceph-osd/tasks/start_osds.yml index 15ed25d79..e91f3723c 100644 --- a/roles/ceph-osd/tasks/start_osds.yml +++ b/roles/ceph-osd/tasks/start_osds.yml @@ -33,13 +33,13 @@ - name: activate containerized osd(s) shell: | - DOCKER_ENV=$(docker run --rm --net=host --ulimit nofile=1024:1024 \ + DOCKER_ENV=$(docker run --rm --net=host --ulimit nofile=1024:4096 \ --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph:z \ -e CLUSTER={{ cluster }} -e OSD_DEVICE={{ item }} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ disk_list) docker run --rm --net=host \ - --ulimit nofile=1024:1024 \ + --ulimit nofile=1024:4096 \ --ipc=host --pid=host --privileged=true \ -v /etc/ceph:/etc/ceph:z \ -v /var/lib/ceph/:/var/lib/ceph/:z \ @@ -80,7 +80,7 @@ - name: set_fact docker_exec_start_osd set_fact: - docker_exec_start_osd: "{{ 'docker run --rm --ulimit nofile=1024:1024 --privileged=true -v /run/lvm/lvmetad.socket:/run/lvm/lvmetad.socket -v /var/run/udev/:/var/run/udev/:z -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}" + docker_exec_start_osd: "{{ 'docker run --rm --ulimit nofile=1024:4096 --privileged=true -v /run/lvm/lvmetad.socket:/run/lvm/lvmetad.socket -v /var/run/udev/:/var/run/udev/:z -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}" when: osd_scenario == 'lvm' - name: collect osd ids diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 index e7e0e29cf..b2d57d3c2 100644 --- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 +++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 @@ -14,9 +14,9 @@ DOCKER_ENV="" ############# function id_to_device () { {% if dmcrypt | bool %} - docker run --rm --net=host --ulimit nofile=1024:1024 --ipc=host --pid=host --privileged=true -v /etc/ceph:/etc/ceph:z -v /var/lib/ceph/:/var/lib/ceph/:z -v /dev:/dev -v /etc/localtime:/etc/localtime:ro -e DEBUG=verbose -e CLUSTER={{ cluster }} {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} osd_ceph_disk_dmcrypt_data_map + docker run --rm --net=host --ulimit nofile=1024:4096 --ipc=host --pid=host --privileged=true -v /etc/ceph:/etc/ceph:z -v /var/lib/ceph/:/var/lib/ceph/:z -v /dev:/dev -v /etc/localtime:/etc/localtime:ro -e DEBUG=verbose -e CLUSTER={{ cluster }} {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} osd_ceph_disk_dmcrypt_data_map {% endif %} - DATA_PART=$(docker run --rm --ulimit nofile=1024:1024 --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph:z --entrypoint ceph-disk {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} list | grep ", osd\.${1}," | awk '{ print $1 }') + DATA_PART=$(docker run --rm --ulimit nofile=1024:4096 --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph:z --entrypoint ceph-disk {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} list | grep ", osd\.${1}," | awk '{ print $1 }') if [[ "${DATA_PART}" =~ ^/dev/(cciss|nvme) ]]; then OSD_DEVICE=${DATA_PART:0:-2} else @@ -94,6 +94,7 @@ numactl \ --privileged=true \ --pid=host \ --ipc=host \ + --ulimit nofile=1024:4096 \ {% if osd_objectstore == 'filestore' -%} --memory={{ ceph_osd_docker_memory_limit }} \ {% endif -%}