fix(kubernetes): taint nodes with kubectl (#10705)

Signed-off-by: Maxime Leroy <19607336+maxime1907@users.noreply.github.com>
pull/10836/head
Maxime Leroy 2024-01-23 15:46:13 +01:00 committed by GitHub
parent 2eb588bed9
commit ab0163a3ad
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 41 additions and 12 deletions

View File

@ -245,7 +245,7 @@ node_labels:
label2_name: label2_value
```
* *node_taints* - Taints applied to nodes via kubelet --register-with-taints parameter.
* *node_taints* - Taints applied to nodes via `kubectl taint node`.
For example, taints can be set in the inventory as variables or more widely in group_vars.
*node_taints* has to be defined as a list of strings in format `key=value:effect`, e.g.:

View File

@ -48,6 +48,7 @@
- { role: kubespray-defaults }
- { role: kubernetes/kubeadm, tags: kubeadm}
- { role: kubernetes/node-label, tags: node-label }
- { role: kubernetes/node-taint, tags: node-taint }
- { role: network_plugin, tags: network }
- { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver }

View File

@ -91,6 +91,7 @@
- { role: kubespray-defaults }
- { role: kubernetes/kubeadm, tags: kubeadm }
- { role: kubernetes/node-label, tags: node-label }
- { role: kubernetes/node-taint, tags: node-taint }
- { role: network_plugin, tags: network }
- name: Apply resolv.conf changes now that cluster DNS is up

View File

@ -55,6 +55,7 @@
- { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true }
- { role: kubernetes/client, tags: client }
- { role: kubernetes/node-label, tags: node-label }
- { role: kubernetes/node-taint, tags: node-taint }
- { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
- { role: kubernetes-apps, tags: csi-driver }
- { role: upgrade/post-upgrade, tags: post-upgrade }
@ -87,6 +88,7 @@
- { role: kubernetes/node, tags: node }
- { role: kubernetes/kubeadm, tags: kubeadm }
- { role: kubernetes/node-label, tags: node-label }
- { role: kubernetes/node-taint, tags: node-taint }
- { role: upgrade/post-upgrade, tags: post-upgrade }
- name: Patch Kubernetes for Windows

View File

@ -0,0 +1,35 @@
---
- name: Set role and inventory node taint to empty list
set_fact:
role_node_taints: []
inventory_node_taints: []
- name: Node taint for nvidia GPU nodes
set_fact:
role_node_taints: "{{ role_node_taints + ['nvidia.com/gpu=:NoSchedule'] }}"
when:
- nvidia_gpu_nodes is defined
- nvidia_accelerator_enabled | bool
- inventory_hostname in nvidia_gpu_nodes
- name: Populate inventory node taint
set_fact:
inventory_node_taints: "{{ inventory_node_taints + ['%s' | format(item)] }}"
loop: "{{ node_taints | d([]) }}"
when:
- node_taints is defined
- node_taints is not string
- node_taints is not mapping
- node_taints is iterable
- debug: # noqa name[missing]
var: role_node_taints
- debug: # noqa name[missing]
var: inventory_node_taints
- name: Set taint to node
command: >-
{{ kubectl }} taint node {{ kube_override_hostname | default(inventory_hostname) }} {{ (role_node_taints + inventory_node_taints) | join(' ') }} --overwrite=true
delegate_to: "{{ groups['kube_control_plane'][0] }}"
changed_when: false
when:
- (role_node_taints + inventory_node_taints) | length > 0

View File

@ -15,17 +15,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
--runtime-cgroups={{ kubelet_runtime_cgroups }} \
{% endset %}
{# Kubelet node taints for gpu #}
{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
{% if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %}
{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
{% elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %}
{% set node_taints = [] %}
{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
{% endif %}
{% endif %}
KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
KUBELET_ARGS="{{ kubelet_args_base }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
{% if kubelet_flexvolumes_plugins_dir is defined %}
KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
{% endif %}