From 9f45552201ea4eed09260787b65dc8853042d4ac Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 23 Sep 2024 14:07:28 +0200 Subject: [PATCH 1/4] Cleanup redundancy k8s_cluster = kube_control_plane + kube_node --- roles/kubernetes/node/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 1df9d6418..a0ab20854 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -28,7 +28,7 @@ kubelet_systemd_wants_dependencies: [] # List of secure IPs for kubelet kube_node_addresses: >- - {%- for host in (groups['kube_control_plane'] + groups['kube_node'] + groups['etcd']) | unique -%} + {%- for host in (groups['k8s_cluster'] | union(groups['etcd'])) -%} {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ ' ' if not loop.last else '' }} {%- endfor -%} kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnet }} {{ kube_node_addresses }}" From dba00f2d85f782fea9bc4aae9b0c434df97ac959 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 23 Sep 2024 15:30:48 +0200 Subject: [PATCH 2/4] metrics-server: Unconditional control-plane tolerations There is no harm on having unneeded toleration when control-plane node are not tainted, so simplify the template to always use the toleration. --- roles/kubernetes-apps/metrics_server/tasks/main.yml | 5 ----- .../templates/metrics-server-deployment.yaml.j2 | 4 ---- 2 files changed, 9 deletions(-) diff --git a/roles/kubernetes-apps/metrics_server/tasks/main.yml b/roles/kubernetes-apps/metrics_server/tasks/main.yml index 9c4e7cb7c..1a32fdab9 100644 --- a/roles/kubernetes-apps/metrics_server/tasks/main.yml +++ b/roles/kubernetes-apps/metrics_server/tasks/main.yml @@ -1,9 +1,4 @@ --- -# If all control plane nodes have the node role, there are no tainted control plane nodes and toleration should not be specified. -- name: Check all control plane nodes are node or not - set_fact: - control_plane_nodes_are_not_tainted: "{{ groups['kube_node'] | intersect(groups['kube_control_plane']) == groups['kube_control_plane'] }}" - - name: Metrics Server | Delete addon dir file: path: "{{ kube_config_dir }}/addons/metrics_server" diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 index 746af17ad..a7052f9be 100644 --- a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 +++ b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 @@ -85,15 +85,11 @@ spec: volumes: - name: tmp emptyDir: {} -{% if not control_plane_nodes_are_not_tainted or metrics_server_extra_tolerations is defined %} tolerations: -{% if not control_plane_nodes_are_not_tainted %} - key: node-role.kubernetes.io/control-plane effect: NoSchedule -{% endif %} {% if metrics_server_extra_tolerations is defined %} {{ metrics_server_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} -{% endif %} {% endif %} affinity: podAntiAffinity: From 0679d9c8e940885174db6cd9dcd8ccd9351d1505 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 23 Sep 2024 15:42:24 +0200 Subject: [PATCH 3/4] metrics-server: proper defaults, cleaner deploy template --- .../metrics_server/defaults/main.yml | 6 +++--- .../templates/metrics-server-deployment.yaml.j2 | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/roles/kubernetes-apps/metrics_server/defaults/main.yml b/roles/kubernetes-apps/metrics_server/defaults/main.yml index 934e67b75..ed43734bb 100644 --- a/roles/kubernetes-apps/metrics_server/defaults/main.yml +++ b/roles/kubernetes-apps/metrics_server/defaults/main.yml @@ -9,6 +9,6 @@ metrics_server_requests_cpu: 100m metrics_server_requests_memory: 200Mi metrics_server_host_network: false metrics_server_replicas: 1 -# metrics_server_extra_tolerations: [] -# metrics_server_extra_affinity: {} -# metrics_server_nodeselector: {} +metrics_server_extra_tolerations: [] +metrics_server_extra_affinity: {} +metrics_server_nodeselector: {} diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 index a7052f9be..528e23a13 100644 --- a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 +++ b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 @@ -88,9 +88,9 @@ spec: tolerations: - key: node-role.kubernetes.io/control-plane effect: NoSchedule -{% if metrics_server_extra_tolerations is defined %} +{% if metrics_server_extra_tolerations %} {{ metrics_server_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} -{% endif %} +{%- endif %} affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -105,10 +105,10 @@ spec: topologyKey: kubernetes.io/hostname namespaces: - kube-system -{% if metrics_server_extra_affinity is defined %} - {{ metrics_server_extra_affinity | to_nice_yaml | indent(width=8) }} -{% endif %} -{% if metrics_server_nodeselector is defined %} +{% if metrics_server_extra_affinity %} + {{ metrics_server_extra_affinity | to_nice_yaml(indent=2) | indent(8) }} +{%- endif %} +{% if metrics_server_nodeselector %} nodeSelector: - {{ metrics_server_nodeselector | to_nice_yaml | indent(width=8) }} -{% endif %} + {{ metrics_server_nodeselector | to_nice_yaml(indent=2) | indent(8) }} +{%- endif %} From fbcc8cc336aeddb83605824de3f304067c783106 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 23 Sep 2024 16:09:07 +0200 Subject: [PATCH 4/4] control-plane: refactor group membership test Testing with group_names does not require the groups to exist. --- .../control-plane/templates/kubeadm-config.v1beta3.yaml.j2 | 2 +- .../templates/kubeadm-controlplane.v1beta3.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index 9dd5e4376..e929a8436 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -16,7 +16,7 @@ nodeRegistration: {% if kube_override_hostname | default('') %} name: "{{ kube_override_hostname }}" {% endif %} -{% if inventory_hostname in groups['kube_control_plane'] and inventory_hostname not in groups['kube_node'] %} +{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %} taints: - effect: NoSchedule key: node-role.kubernetes.io/control-plane diff --git a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 index bc9f3bdf9..59759188d 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 @@ -24,7 +24,7 @@ controlPlane: nodeRegistration: name: {{ kube_override_hostname | default(inventory_hostname) }} criSocket: {{ cri_socket }} -{% if inventory_hostname in groups['kube_control_plane'] and inventory_hostname not in groups['kube_node'] %} +{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %} taints: - effect: NoSchedule key: node-role.kubernetes.io/control-plane