diff --git a/docs/ansible/vars.md b/docs/ansible/vars.md index f8d040e12..4768c536a 100644 --- a/docs/ansible/vars.md +++ b/docs/ansible/vars.md @@ -296,8 +296,8 @@ node_taints: For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. -Extra flags for the kubelet can be specified using these variables, -in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubelet YAML config file. The `kubelet_node_config_extra_args` apply kubelet settings only to nodes and not control planes. Example: +Extra flags for the kubelet can be specified using these variables, in the form of dicts of key-value pairs of +configuration parameters that will be inserted into the kubelet YAML config file. Example: ```yml kubelet_config_extra_args: @@ -312,14 +312,10 @@ kubelet_config_extra_args: The possible vars are: * *kubelet_config_extra_args* -* *kubelet_node_config_extra_args* Previously, the same parameters could be passed as flags to kubelet binary with the following vars: * *kubelet_custom_flags* -* *kubelet_node_custom_flags* - -The `kubelet_node_custom_flags` apply kubelet settings only to nodes and not control planes. Example: ```yml kubelet_custom_flags: diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 4d4b011a7..1df9d6418 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -36,31 +36,20 @@ kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnet }} {{ kube_n # Reserve this space for kube resources # Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations). kube_reserved: false -kube_reserved_cgroups_for_service_slice: kube.slice kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}" -kube_memory_reserved: 256Mi -kube_cpu_reserved: 100m -# kube_ephemeral_storage_reserved: 2Gi -# kube_pid_reserved: "1000" -# Reservation for control plane hosts -kube_master_memory_reserved: 512Mi -kube_master_cpu_reserved: 200m -# kube_master_ephemeral_storage_reserved: 2Gi -# kube_master_pid_reserved: "1000" +kube_memory_reserved: "256Mi" +kube_cpu_reserved: "100m" +kube_ephemeral_storage_reserved: "500Mi" +kube_pid_reserved: "1000" # Set to true to reserve resources for system daemons system_reserved: false system_reserved_cgroups_for_service_slice: system.slice system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}" -system_memory_reserved: 512Mi -system_cpu_reserved: 500m -# system_ephemeral_storage_reserved: 2Gi -# system_pid_reserved: "1000" -# Reservation for control plane hosts -system_master_memory_reserved: 256Mi -system_master_cpu_reserved: 250m -# system_master_ephemeral_storage_reserved: 2Gi -# system_master_pid_reserved: "1000" +system_memory_reserved: "512Mi" +system_cpu_reserved: "500m" +system_ephemeral_storage_reserved: "500Mi" +system_pid_reserved: 1000 ## Eviction Thresholds to avoid system OOMs # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds @@ -136,9 +125,6 @@ kubelet_config_extra_args_cgroupfs: systemCgroups: /system.slice cgroupRoot: / -## Support parameters to be passed to kubelet via kubelet-config.yaml only on nodes, not control plane nodes -kubelet_node_config_extra_args: {} - # Maximum number of container log files that can be present for a container. kubelet_logfiles_max_nr: 5 @@ -148,9 +134,6 @@ kubelet_logfiles_max_size: 10Mi ## Support custom flags to be passed to kubelet kubelet_custom_flags: [] -## Support custom flags to be passed to kubelet only on nodes, not control plane nodes -kubelet_node_custom_flags: [] - # If non-empty, will use this string as identification instead of the actual hostname kube_override_hostname: >- {%- if cloud_provider is defined and cloud_provider in ['aws'] -%} diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 index 325cc2fa1..0474bf9bb 100644 --- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -61,56 +61,16 @@ clusterDNS: - {{ dns_address }} {% endfor %} {# Node reserved CPU/memory #} -{% if kube_reserved | bool %} -kubeReservedCgroup: {{ kube_reserved_cgroups }} +{% for scope in "kube", "system" %} +{% if lookup('ansible.builtin.vars', scope + "_reserved") | bool %} +{{ scope }}ReservedCgroup: {{ lookup('ansible.builtin.vars', scope + '_reserved_cgroups') }} {% endif %} -kubeReserved: -{% if 'kube_control_plane' in group_names %} - cpu: "{{ kube_master_cpu_reserved }}" - memory: {{ kube_master_memory_reserved }} -{% if kube_master_ephemeral_storage_reserved is defined %} - ephemeral-storage: {{ kube_master_ephemeral_storage_reserved }} -{% endif %} -{% if kube_master_pid_reserved is defined %} - pid: "{{ kube_master_pid_reserved }}" -{% endif %} -{% else %} - cpu: "{{ kube_cpu_reserved }}" - memory: {{ kube_memory_reserved }} -{% if kube_ephemeral_storage_reserved is defined %} - ephemeral-storage: {{ kube_ephemeral_storage_reserved }} -{% endif %} -{% if kube_pid_reserved is defined %} - pid: "{{ kube_pid_reserved }}" -{% endif %} -{% endif %} -{% if system_reserved | bool %} -systemReservedCgroup: {{ system_reserved_cgroups }} -systemReserved: -{% if 'kube_control_plane' in group_names %} - cpu: "{{ system_master_cpu_reserved }}" - memory: {{ system_master_memory_reserved }} -{% if system_master_ephemeral_storage_reserved is defined %} - ephemeral-storage: {{ system_master_ephemeral_storage_reserved }} -{% endif %} -{% if system_master_pid_reserved is defined %} - pid: "{{ system_master_pid_reserved }}" -{% endif %} -{% else %} - cpu: "{{ system_cpu_reserved }}" - memory: {{ system_memory_reserved }} -{% if system_ephemeral_storage_reserved is defined %} - ephemeral-storage: {{ system_ephemeral_storage_reserved }} -{% endif %} -{% if system_pid_reserved is defined %} - pid: "{{ system_pid_reserved }}" -{% endif %} -{% endif %} -{% endif %} -{% if ('kube_control_plane' in group_names) and (eviction_hard_control_plane is defined) and eviction_hard_control_plane %} -evictionHard: - {{ eviction_hard_control_plane | to_nice_yaml(indent=2) | indent(2) }} -{% elif ('kube_control_plane' not in group_names) and (eviction_hard is defined) and eviction_hard %} +{{ scope }}Reserved: +{% for resource in "cpu", "memory", "ephemeral-storage", "pid" %} + {{ resource }}: "{{ lookup('ansible.builtin.vars', scope + '_' ~ (resource | replace('-', '_')) + '_reserved') }}" +{% endfor %} +{% endfor %} +{% if eviction_hard is defined and eviction_hard %} evictionHard: {{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }} {% endif %} @@ -118,9 +78,6 @@ resolvConf: "{{ kube_resolv_conf }}" {% if kubelet_config_extra_args %} {{ kubelet_config_extra_args | to_nice_yaml(indent=2) }} {% endif %} -{% if inventory_hostname in groups['kube_node'] and kubelet_node_config_extra_args %} -{{ kubelet_node_config_extra_args | to_nice_yaml(indent=2) }} -{% endif %} {% if kubelet_feature_gates or kube_feature_gates %} featureGates: {% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %} diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 index 643194bf1..576b9c8fd 100644 --- a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -14,7 +14,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --runtime-cgroups={{ kubelet_runtime_cgroups }} \ {% endset %} -KUBELET_ARGS="{{ kubelet_args_base }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_custom_flags | join(' ') }}" {% if kubelet_flexvolumes_plugins_dir is defined %} KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}" {% endif %} diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index 71c88d817..7d36da08a 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -29,6 +29,9 @@ kube_proxy_mode: ipvs ## The timeout for init first control-plane kubeadm_init_timeout: 300s +# TODO: remove this +kube_reserved_cgroups_for_service_slice: kube.slice + ## List of kubeadm init phases that should be skipped during control plane setup ## By default 'addon/coredns' is skipped ## 'addon/kube-proxy' gets skipped for some network plugins diff --git a/tests/files/packet_ubuntu24-calico-all-in-one.yml b/tests/files/packet_ubuntu24-calico-all-in-one.yml index 5d7f55878..310bf349f 100644 --- a/tests/files/packet_ubuntu24-calico-all-in-one.yml +++ b/tests/files/packet_ubuntu24-calico-all-in-one.yml @@ -22,3 +22,6 @@ containerd_registries_mirrors: - host: http://172.19.16.11:5000 capabilities: ["pull", "resolve", "push"] skip_verify: true + +kube_reserved: true +system_reserved: true