add host-based calculation for kube cpu/memory reserved resources

pull/11082/head
Payback159 2024-09-24 19:07:50 +02:00
parent ebdc599b05
commit fd1ae56def
4 changed files with 31 additions and 5 deletions

View File

@ -70,4 +70,17 @@ After the setup, the cgroups hierarchy is as follows:
└── ...
```
## Automatic Resource Reservation Calculation
While manually specifying resource reservations for kube and system daemons works, Kubespray offers a more convenient approach. You can set the `kube_enable_auto_reserved_resources` flag to `true`. This instructs Kubespray to automatically calculate appropriate resource reservations for `kube_[master]_cpu_reserved` and `kube_[master]_memory_reserved` based on your host size. This eliminates the need for manual configuration, simplifying the process.
When `kube_enable_auto_reserved_resources` is set to `true`, Kubespray calculates resource reservations as follows:
CPU: 1% of the total CPU cores on the host machine + 80 millicores
Memory: 5% of the total available system memory + 330 Megabytes
This approach ensures that kubelet has sufficient resources to function properly while leaving the majority of resources available for your workloads.
After the setup, the cgroups hierarchy remains the same as before.
You can learn more in the [official kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/).

View File

@ -264,6 +264,10 @@ default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
# Whether to run kubelet and container-engine daemons in a dedicated cgroup.
# kube_reserved: false
# If kube_enable_auto_reserved_resources is true,
# the cpu and memory values are calculated based on the number of CPUs and memory size of the host.
# The calculation formulas can be seen in https://github.com/kubernetes-sigs/kubespray/blob/master/docs/cgroups.md
# kube_enable_auto_reserved_resources: false
## Uncomment to override default values
## The following two items need to be set when kube_reserved is true
# kube_reserved_cgroups_for_service_slice: kube.slice

View File

@ -9,7 +9,7 @@ kubelet_bind_address: "{{ ip | default('0.0.0.0') }}"
kube_resolv_conf: "/etc/resolv.conf"
# Set to empty to avoid cgroup creation
kubelet_enforce_node_allocatable: "\"\""
kubelet_enforce_node_allocatable: '""'
# Set runtime and kubelet cgroups when using systemd as cgroup driver (default)
kube_service_cgroups: "{% if kube_reserved %}{{ kube_reserved_cgroups_for_service_slice }}{% else %}system.slice{% endif %}"
@ -32,13 +32,14 @@ kube_node_addresses: >-
{{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ ' ' if not loop.last else '' }}
{%- endfor -%}
kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnet }} {{ kube_node_addresses }}"
# Reserve this space for kube resources
# Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations).
kube_reserved: false
kube_enable_auto_reserved_resources: false
kube_reserved_cgroups_for_service_slice: kube.slice
kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}"
kube_memory_reserved: "256Mi"
kube_cpu_reserved: "100m"
kube_memory_reserved: "{{ kube_enable_auto_reserved_resources | ternary((ansible_memtotal_mb * 0.05 + 330) | int | string ~ 'Mi', '256Mi') }}"
kube_cpu_reserved: "{{ kube_enable_auto_reserved_resources | ternary((ansible_processor_vcpus * 1000 * 0.01 + 80) | int | string ~ 'm', '100m') }}"
kube_ephemeral_storage_reserved: "500Mi"
kube_pid_reserved: "1000"
@ -249,7 +250,6 @@ conntrack_modules:
- nf_conntrack
- nf_conntrack_ipv4
## Enable distributed tracing for kubelet
kubelet_tracing: false
kubelet_tracing_endpoint: 0.0.0.0:4317

View File

@ -18,6 +18,15 @@
- kubelet
- kubeadm
- name: Gather facts about the node
setup:
filter: ansible_memtotal_mb,ansible_processor_vcpus
tags:
- kubelet
- kubeadm
when:
- kube_enable_auto_reserved_resources
- name: Write kubelet config file
template:
src: "kubelet-config.{{ kubeletConfig_api_version }}.yaml.j2"