container: introduce resource limitation for containers

This can be controlled via 2 options:

* ceph_$DAEMON_docker_memory_limit
* ceph_$DAEMON_docker_cpu_limit

All daemons default to 1GB for memory and 1 CPU by default.
Recommendations from:
https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations

Signed-off-by: Sébastien Han <seb@redhat.com>

group_vars/mdss.yml.sample

@@ -25,6 +25,13 @@ dummy:
 # DOCKER #
 ##########
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_mds_docker_extra_env' variable.
+#ceph_mds_docker_memory_limit: 1g
+#ceph_mds_docker_cpu_limit: 1
+
 #ceph_mds_docker_extra_env: -e CLUSTER={{ cluster }} -e MDS_NAME={{ ansible_hostname }}
 #ceph_config_keys: [] # DON'T TOUCH ME
 

group_vars/mgrs.yml.sample

@@ -11,6 +11,13 @@ dummy:
 # DOCKER #
 ##########
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_mgr_docker_extra_env' variable.
+#ceph_mgr_docker_memory_limit: 1g
+#ceph_mgr_docker_cpu_limit: 1
+
 #ceph_mgr_docker_extra_env: -e CLUSTER={{ cluster }} -e MGR_NAME={{ ansible_hostname }}
 #ceph_config_keys: [] # DON'T TOUCH ME
 

group_vars/mons.yml.sample

@@ -115,6 +115,13 @@ dummy:
 ##########
 #ceph_mon_docker_subnet: "{{ public_network }}"# subnet of the monitor_interface
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_mon_docker_extra_env' variable.
+#ceph_mon_docker_memory_limit: 1g
+#ceph_mon_docker_cpu_limit: 1
+
 # ceph_mon_docker_extra_env:
 #
 # Use this variable to add extra env configuration to run your mon container.

group_vars/osds.yml.sample

@@ -226,6 +226,13 @@ dummy:
 
 #ceph_config_keys: [] # DON'T TOUCH ME
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_osd_docker_extra_env' variable.
+#ceph_osd_docker_memory_limit: 1g
+#ceph_osd_docker_cpu_limit: 1
+
 # PREPARE DEVICE
 #
 # WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above

group_vars/rbd-mirrors.yml.sample

@@ -23,6 +23,7 @@ dummy:
 # NOTE: deprecated generic local user id for pre-Luminous releases
 #ceph_rbd_mirror_local_user: "admin"
 
+
 #################
 # CONFIGURATION #
 #################
@@ -41,10 +42,19 @@ dummy:
 # /etc/ceph/<remote_cluster>.client.<remote_user>.keyring
 #ceph_rbd_mirror_remote_user: ""
 
+
 ##########
 # DOCKER #
 ##########
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_rbd_mirror_docker_extra_env' variable.
+#ceph_rbd_mirror_docker_memory_limit: 1g
+#ceph_rbd_mirror_docker_cpu_limit: 1
+
+#ceph_rbd_mirror_docker_extra_env:
 #ceph_config_keys: [] # DON'T TOUCH ME
 
 

group_vars/rgws.yml.sample

@@ -57,6 +57,13 @@ dummy:
 # DOCKER #
 ##########
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_rgw_docker_extra_env' variable.
+#ceph_rgw_docker_memory_limit: 1g
+#ceph_rgw_docker_cpu_limit: 1
+
 #ceph_rgw_docker_extra_env:
 #ceph_config_keys: [] # DON'T TOUCH ME
 #rgw_config_keys: "/" # DON'T TOUCH ME

roles/ceph-docker-common/tasks/main.yml

@@ -22,6 +22,15 @@
     monitor_name: "{{ ansible_fqdn }}"
   when: mon_use_fqdn
 
+- name: get docker version
+  command: docker --version
+  changed_when: false
+  always_run: yes
+  register: docker_version
+
+- set_fact:
+    docker_version: "{{ docker_version.stdout.split(' ')[2] }}"
+
 - name: check if a cluster is already running
   command: "docker ps -q -a --filter='ancestor={{ ceph_docker_image }}:{{ ceph_docker_image_tag }}'"
   register: ceph_health

roles/ceph-mds/defaults/main.yml

@@ -17,6 +17,13 @@ copy_admin_key: false
 # DOCKER #
 ##########
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_mds_docker_extra_env' variable.
+ceph_mds_docker_memory_limit: 1g
+ceph_mds_docker_cpu_limit: 1
+
 ceph_mds_docker_extra_env: -e CLUSTER={{ cluster }} -e MDS_NAME={{ ansible_hostname }}
 ceph_config_keys: [] # DON'T TOUCH ME
 

roles/ceph-mds/templates/ceph-mds.service.j2

@@ -7,20 +7,26 @@ EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker stop ceph-mds-{{ ansible_hostname }}
 ExecStartPre=-/usr/bin/docker rm ceph-mds-{{ ansible_hostname }}
 ExecStart=/usr/bin/docker run --rm --net=host \
-   {% if not containerized_deployment_with_kv -%}
-   -v /var/lib/ceph:/var/lib/ceph \
-   -v /etc/ceph:/etc/ceph \
-   {% else -%}
-   -e KV_TYPE={{kv_type}} \
-   -e KV_IP={{kv_endpoint}} \
-   -e KV_PORT={{kv_port}} \
-   {% endif -%}
-   -v /etc/localtime:/etc/localtime:ro \
-   -e CEPH_DAEMON=MDS \
-   -e CEPHFS_CREATE=1 \
-   {{ ceph_mds_docker_extra_env }} \
-   --name=ceph-mds-{{ ansible_hostname }} \
-   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
+  --memory={{ ceph_mds_docker_memory_limit }} \
+  {% if docker_version.split('.')[0] | version_compare('13', '>=') -%}
+  --cpus={{ ceph_mds_docker_cpu_limit }} \
+  {% else -%}
+  --cpu-quota={{ ceph_mds_docker_cpu_limit * 100000 }} \
+  {% endif -%}
+  {% if not containerized_deployment_with_kv -%}
+  -v /var/lib/ceph:/var/lib/ceph \
+  -v /etc/ceph:/etc/ceph \
+  {% else -%}
+  -e KV_TYPE={{kv_type}} \
+  -e KV_IP={{kv_endpoint}} \
+  -e KV_PORT={{kv_port}} \
+  {% endif -%}
+  -v /etc/localtime:/etc/localtime:ro \
+  -e CEPH_DAEMON=MDS \
+  -e CEPHFS_CREATE=1 \
+  {{ ceph_mds_docker_extra_env }} \
+  --name=ceph-mds-{{ ansible_hostname }} \
+  {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
 ExecStopPost=-/usr/bin/docker stop ceph-mds-{{ ansible_hostname }}
 Restart=always
 RestartSec=10s

roles/ceph-mgr/defaults/main.yml

@@ -3,6 +3,13 @@
 # DOCKER #
 ##########
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_mgr_docker_extra_env' variable.
+ceph_mgr_docker_memory_limit: 1g
+ceph_mgr_docker_cpu_limit: 1
+
 ceph_mgr_docker_extra_env: -e CLUSTER={{ cluster }} -e MGR_NAME={{ ansible_hostname }}
 ceph_config_keys: [] # DON'T TOUCH ME
 

roles/ceph-mgr/templates/ceph-mgr.service.j2

@@ -7,19 +7,25 @@ EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker stop ceph-mgr-{{ ansible_hostname }}
 ExecStartPre=-/usr/bin/docker rm ceph-mgr-{{ ansible_hostname }}
 ExecStart=/usr/bin/docker run --rm --net=host \
-   {% if not containerized_deployment_with_kv -%}
-   -v /var/lib/ceph:/var/lib/ceph \
-   -v /etc/ceph:/etc/ceph \
-   {% else -%}
-   -e KV_TYPE={{kv_type}} \
-   -e KV_IP={{kv_endpoint}} \
-   -e KV_PORT={{kv_port}} \
-   {% endif -%}
-   -v /etc/localtime:/etc/localtime:ro \
-   -e CEPH_DAEMON=MGR \
-   {{ ceph_mgr_docker_extra_env }} \
-   --name=ceph-mgr-{{ ansible_hostname }} \
-   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
+  --memory={{ ceph_mgr_docker_memory_limit }} \
+  {% if docker_version.split('.')[0] | version_compare('13', '>=') -%}
+  --cpus={{ ceph_mgr_docker_cpu_limit }} \
+  {% else -%}
+  --cpu-quota={{ ceph_mgr_docker_cpu_limit * 100000 }} \
+  {% endif -%}
+  {% if not containerized_deployment_with_kv -%}
+  -v /var/lib/ceph:/var/lib/ceph \
+  -v /etc/ceph:/etc/ceph \
+  {% else -%}
+  -e KV_TYPE={{kv_type}} \
+  -e KV_IP={{kv_endpoint}} \
+  -e KV_PORT={{kv_port}} \
+  {% endif -%}
+  -v /etc/localtime:/etc/localtime:ro \
+  -e CEPH_DAEMON=MGR \
+  {{ ceph_mgr_docker_extra_env }} \
+  --name=ceph-mgr-{{ ansible_hostname }} \
+  {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
 ExecStopPost=-/usr/bin/docker stop ceph-mgr-{{ ansible_hostname }}
 Restart=always
 RestartSec=10s

roles/ceph-mon/defaults/main.yml

@@ -107,6 +107,13 @@ openstack_keys:
 ##########
 ceph_mon_docker_subnet: "{{ public_network }}"# subnet of the monitor_interface
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_mon_docker_extra_env' variable.
+ceph_mon_docker_memory_limit: 1g
+ceph_mon_docker_cpu_limit: 1
+
 # ceph_mon_docker_extra_env:
 #
 # Use this variable to add extra env configuration to run your mon container.

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -7,37 +7,43 @@ EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker rm ceph-mon-%i
 ExecStartPre=$(command -v mkdir) -p /etc/ceph /var/lib/ceph/mon
 ExecStart=/usr/bin/docker run --rm --name ceph-mon-%i --net=host \
-   {% if not containerized_deployment_with_kv -%}
-   -v /var/lib/ceph:/var/lib/ceph \
-   -v /etc/ceph:/etc/ceph \
-   {% else -%}
-   -e KV_TYPE={{kv_type}} \
-   -e KV_IP={{kv_endpoint}}\
-   -e KV_PORT={{kv_port}} \
-   {% endif -%}
-   -v /etc/localtime:/etc/localtime:ro \
-   {% if mon_docker_privileged -%}
-   --privileged \
-   {% endif -%}
-   {% if mon_docker_net_host -%}
-   --net=host \
-   {% endif -%}
-   -e CEPH_DAEMON=MON \
-   -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \
-   -e IP_VERSION={{ ip_version[-1:] }} \
-   {% if monitor_address is defined and monitor_address != '0.0.0.0' %}
-     {% if ip_version == 'ipv4' -%}
-       -e MON_IP={{ monitor_address }} \
-     {% elif ip_version == 'ipv6' -%}
-       -e MON_IP=[{{ monitor_address }}] \
-     {% endif -%}
-   {% elif ip_version == 'ipv4' -%}
-   -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + monitor_interface][ip_version]['address'] }} \
-   {% elif ip_version =='ipv6' -%}
-   -e MON_IP=[{{ hostvars[inventory_hostname]['ansible_' + monitor_interface][ip_version][0]['address'] }}] \
-   {% endif -%}
-   {{ ceph_mon_docker_extra_env }} \
-   {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
+  --memory={{ ceph_mon_docker_memory_limit }} \
+  {% if docker_version.split('.')[0] | version_compare('13', '>=') -%}
+  --cpus={{ ceph_mon_docker_cpu_limit }} \
+  {% else -%}
+  --cpu-quota={{ ceph_mon_docker_cpu_limit * 100000 }} \
+  {% endif -%}
+  {% if not containerized_deployment_with_kv -%}
+  -v /var/lib/ceph:/var/lib/ceph \
+  -v /etc/ceph:/etc/ceph \
+  {% else -%}
+  -e KV_TYPE={{kv_type}} \
+  -e KV_IP={{kv_endpoint}}\
+  -e KV_PORT={{kv_port}} \
+  {% endif -%}
+  -v /etc/localtime:/etc/localtime:ro \
+  {% if mon_docker_privileged -%}
+  --privileged \
+  {% endif -%}
+  {% if mon_docker_net_host -%}
+  --net=host \
+  {% endif -%}
+  -e CEPH_DAEMON=MON \
+  -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \
+  -e IP_VERSION={{ ip_version[-1:] }} \
+  {% if monitor_address is defined and monitor_address != '0.0.0.0' %}
+    {% if ip_version == 'ipv4' -%}
+      -e MON_IP={{ monitor_address }} \
+    {% elif ip_version == 'ipv6' -%}
+      -e MON_IP=[{{ monitor_address }}] \
+    {% endif -%}
+  {% elif ip_version == 'ipv4' -%}
+  -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + monitor_interface][ip_version]['address'] }} \
+  {% elif ip_version =='ipv6' -%}
+  -e MON_IP=[{{ hostvars[inventory_hostname]['ansible_' + monitor_interface][ip_version][0]['address'] }}] \
+  {% endif -%}
+  {{ ceph_mon_docker_extra_env }} \
+  {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
 ExecStopPost=-/usr/bin/docker stop ceph-mon-%i
 Restart=always
 RestartSec=10s

roles/ceph-osd/defaults/main.yml

@@ -218,6 +218,13 @@ lvm_volumes: []
 
 ceph_config_keys: [] # DON'T TOUCH ME
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_osd_docker_extra_env' variable.
+ceph_osd_docker_memory_limit: 1g
+ceph_osd_docker_cpu_limit: 1
+
 # PREPARE DEVICE
 #
 # WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above

roles/ceph-osd/templates/ceph-osd-run.sh.j2

@@ -54,6 +54,12 @@ expose_partitions $1
   --net=host \
   --privileged=true \
   --pid=host \
+  --memory={{ ceph_osd_docker_memory_limit }} \
+  {% if docker_version.split('.')[0] | version_compare('13', '>=') -%}
+  --cpus={{ ceph_osd_docker_cpu_limit }} \
+  {% else -%}
+  --cpu-quota={{ ceph_osd_docker_cpu_limit * 100000 }} \
+  {% endif -%}
   -v /dev:/dev \
   -v /etc/localtime:/etc/localtime:ro \
   -v /var/lib/ceph:/var/lib/ceph \

roles/ceph-rbd-mirror/defaults/main.yml

@@ -15,6 +15,7 @@ copy_admin_key: false
 # NOTE: deprecated generic local user id for pre-Luminous releases
 ceph_rbd_mirror_local_user: "admin"
 
+
 #################
 # CONFIGURATION #
 #################
@@ -33,10 +34,19 @@ ceph_rbd_mirror_remote_cluster: ""
 # /etc/ceph/<remote_cluster>.client.<remote_user>.keyring
 ceph_rbd_mirror_remote_user: ""
 
+
 ##########
 # DOCKER #
 ##########
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_rbd_mirror_docker_extra_env' variable.
+ceph_rbd_mirror_docker_memory_limit: 1g
+ceph_rbd_mirror_docker_cpu_limit: 1
+
+ceph_rbd_mirror_docker_extra_env:
 ceph_config_keys: [] # DON'T TOUCH ME
 
 

roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2

@@ -7,17 +7,24 @@ EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker stop ceph-rbd-mirror-{{ ansible_hostname }}
 ExecStartPre=-/usr/bin/docker rm ceph-rbd-mirror-{{ ansible_hostname }}
 ExecStart=/usr/bin/docker run --rm --net=host \
-   {% if not containerized_deployment_with_kv -%}
-   -v /etc/ceph:/etc/ceph \
-   {% else -%}
-   -e KV_TYPE={{kv_type}} \
-   -e KV_IP={{kv_endpoint}} \
-   -e KV_PORT={{kv_port}} \
-   {% endif -%}
-   -v /etc/localtime:/etc/localtime:ro \
-   -e CEPH_DAEMON=RBD_MIRROR \
-   --name=ceph-rbd-mirror-{{ ansible_hostname }} \
-   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
+  --memory={{ ceph_rbd_mirorr_docker_memory_limit }} \
+  {% if docker_version.split('.')[0] | version_compare('13', '>=') -%}
+  --cpus={{ ceph_rbd_mirror_docker_cpu_limit }} \
+  {% else -%}
+  --cpu-quota={{ ceph_rbd_mirorr_docker_cpu_limit * 100000 }} \
+  {% endif -%}
+  {% if not containerized_deployment_with_kv -%}
+  -v /etc/ceph:/etc/ceph \
+  {% else -%}
+  -e KV_TYPE={{kv_type}} \
+  -e KV_IP={{kv_endpoint}} \
+  -e KV_PORT={{kv_port}} \
+  {% endif -%}
+  -v /etc/localtime:/etc/localtime:ro \
+  -e CEPH_DAEMON=RBD_MIRROR \
+  --name=ceph-rbd-mirror-{{ ansible_hostname }} \
+  {{ ceph_rbd_mirror_docker_extra_env }} \
+  {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
 ExecStopPost=-/usr/bin/docker stop ceph-rbd-mirror-{{ ansible_hostname }}
 Restart=always
 RestartSec=10s

roles/ceph-rgw/defaults/main.yml

@@ -49,6 +49,13 @@ rgw_pull_proto: "http"
 # DOCKER #
 ##########
 
+# Resource limitation
+# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
+# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
+# These options can be passed using the 'ceph_rgw_docker_extra_env' variable.
+ceph_rgw_docker_memory_limit: 1g
+ceph_rgw_docker_cpu_limit: 1
+
 ceph_rgw_docker_extra_env:
 ceph_config_keys: [] # DON'T TOUCH ME
 rgw_config_keys: "/" # DON'T TOUCH ME

roles/ceph-rgw/templates/ceph-radosgw.service.j2

@@ -7,6 +7,12 @@ EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker stop ceph-rgw-{{ ansible_hostname }}
 ExecStartPre=-/usr/bin/docker rm ceph-rgw-{{ ansible_hostname }}
 ExecStart=/usr/bin/docker run --rm --net=host \
+  --memory={{ ceph_rgw_docker_memory_limit }} \
+  {% if docker_version.split('.')[0] | version_compare('13', '>=') -%}
+  --cpus={{ ceph_rgw_docker_cpu_limit }} \
+  {% else -%}
+  --cpu-quota={{ ceph_rgw_docker_cpu_limit * 100000 }} \
+  {% endif -%}
   {% if not containerized_deployment_with_kv -%}
   -v /var/lib/ceph:/var/lib/ceph \
   -v /etc/ceph:/etc/ceph \