From 8abf49ae13242b281da06da2728401f626ab5c23 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Tue, 12 Dec 2023 14:38:36 +0100 Subject: [PATCH] Disable podCIDR allocation from control-plane when using calico (#10639) * Disable control plane allocating podCIDR for nodes when using calico Calico does not use the .spec.podCIDR field for its IP address management. Furthermore, it can false positives from the kube controller manager if kube_network_node_prefix and calico_pool_blocksize are unaligned, which is the case with the default shipped by kubespray. If the subnets obtained from using kube_network_node_prefix are bigger, this would result at some point in the control plane thinking it does not have subnets left for a new node, while calico will work without problems. Explicitely set a default value of false for calico_ipam_host_local to facilitate its use in templates. * Don't default to kube_network_node_prefix for calico_pool_blocksize They have different semantics: kube_network_node_prefix is intended to be the size of the subnet for all pods on a node, while there can be more than on calico block of the specified size (they are allocated on demand). Besides, this commit does not actually change anything, because the current code is buggy: we don't ever default to kube_network_node_prefix, since the variable is defined in the role defaults. --- roles/kubernetes/control-plane/meta/main.yml | 1 + .../templates/kubeadm-config.v1beta3.yaml.j2 | 4 ++++ roles/network_plugin/calico/meta/main.yml | 3 +++ roles/network_plugin/calico/tasks/check.yml | 2 +- roles/network_plugin/calico/tasks/install.yml | 4 ++-- .../network_plugin/calico/templates/calico-config.yml.j2 | 2 +- .../{calico => calico_defaults}/defaults/main.yml | 8 ++++++-- 7 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 roles/network_plugin/calico/meta/main.yml rename roles/network_plugin/{calico => calico_defaults}/defaults/main.yml (97%) diff --git a/roles/kubernetes/control-plane/meta/main.yml b/roles/kubernetes/control-plane/meta/main.yml index 2657006e0..7d793f92f 100644 --- a/roles/kubernetes/control-plane/meta/main.yml +++ b/roles/kubernetes/control-plane/meta/main.yml @@ -9,3 +9,4 @@ dependencies: when: - etcd_deployment_type == "kubeadm" - not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) + - role: network_plugin/calico_defaults diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index dfccb20b2..cbb221823 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -295,11 +295,15 @@ controllerManager: cluster-cidr: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}" {% endif %} service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}" +{% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %} + allocate-node-cidrs: "false" +{% else %} {% if enable_dual_stack_networks %} node-cidr-mask-size-ipv4: "{{ kube_network_node_prefix }}" node-cidr-mask-size-ipv6: "{{ kube_network_node_prefix_ipv6 }}" {% else %} node-cidr-mask-size: "{{ kube_network_node_prefix }}" +{% endif %} {% endif %} profiling: "{{ kube_profiling }}" terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}" diff --git a/roles/network_plugin/calico/meta/main.yml b/roles/network_plugin/calico/meta/main.yml new file mode 100644 index 000000000..15e9b8c40 --- /dev/null +++ b/roles/network_plugin/calico/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: network_plugin/calico_defaults diff --git a/roles/network_plugin/calico/tasks/check.yml b/roles/network_plugin/calico/tasks/check.yml index 2138be87e..de67b154e 100644 --- a/roles/network_plugin/calico/tasks/check.yml +++ b/roles/network_plugin/calico/tasks/check.yml @@ -168,7 +168,7 @@ - name: "Check if inventory match current cluster configuration" assert: that: - - calico_pool_conf.spec.blockSize | int == (calico_pool_blocksize | default(kube_network_node_prefix) | int) + - calico_pool_conf.spec.blockSize | int == calico_pool_blocksize | int - calico_pool_conf.spec.cidr == (calico_pool_cidr | default(kube_pods_subnet)) - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 4aac9df5b..6b293dcb0 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -223,7 +223,7 @@ "name": "{{ calico_pool_name }}", }, "spec": { - "blockSize": {{ calico_pool_blocksize | default(kube_network_node_prefix) }}, + "blockSize": {{ calico_pool_blocksize }}, "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}", "ipipMode": "{{ calico_ipip_mode }}", "vxlanMode": "{{ calico_vxlan_mode }}", @@ -274,7 +274,7 @@ "name": "{{ calico_pool_name }}-ipv6", }, "spec": { - "blockSize": {{ calico_pool_blocksize_ipv6 | default(kube_network_node_prefix_ipv6) }}, + "blockSize": {{ calico_pool_blocksize_ipv6 }}, "cidr": "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}", "ipipMode": "{{ calico_ipip_mode_ipv6 }}", "vxlanMode": "{{ calico_vxlan_mode_ipv6 }}", diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2 index 4012ef784..f1a3f2e76 100644 --- a/roles/network_plugin/calico/templates/calico-config.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-config.yml.j2 @@ -54,7 +54,7 @@ data: "etcd_key_file": "{{ calico_cert_dir }}/key.pem", "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt", {% endif %} - {% if calico_ipam_host_local is defined %} + {% if calico_ipam_host_local %} "ipam": { "type": "host-local", "subnet": "usePodCidr" diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico_defaults/defaults/main.yml similarity index 97% rename from roles/network_plugin/calico/defaults/main.yml rename to roles/network_plugin/calico_defaults/defaults/main.yml index b3c5f809c..8e716498b 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico_defaults/defaults/main.yml @@ -16,14 +16,14 @@ calico_vxlan_mode: Always # valid values are 'Always', 'Never' and 'CrossSubnet calico_cni_pool: true calico_cni_pool_ipv6: true -# add default ippool blockSize (defaults kube_network_node_prefix) +# add default ippool blockSize calico_pool_blocksize: 26 # Calico doesn't support ipip tunneling for the IPv6. calico_ipip_mode_ipv6: Never calico_vxlan_mode_ipv6: Never -# add default ipv6 ippool blockSize (defaults kube_network_node_prefix_ipv6) +# add default ipv6 ippool blockSize calico_pool_blocksize_ipv6: 122 # Calico network backend can be 'bird', 'vxlan' and 'none' @@ -161,6 +161,10 @@ calico_ipam_autoallocateblocks: true # Calico IPAM maxBlocksPerHost, default 0 calico_ipam_maxblocksperhost: 0 +# Calico host local IPAM (use node .spec.podCIDR) + +calico_ipam_host_local: false + # Calico apiserver (only with kdd) calico_apiserver_enabled: false