From 7da3dbcb397f08986f6796e5eed547ff2548344a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Necatican=20Y=C4=B1ld=C4=B1r=C4=B1m?= Date: Mon, 19 Sep 2022 12:14:31 +0300 Subject: [PATCH] Cilium 1.12 Upgrade (#9225) * Drop support for Cilium < 1.10 Signed-off-by: necatican * Synchronize Cilium templates for 1.11.7 Signed-off-by: necatican * Set Cilium v1.12.1 as the default version Signed-off-by: necatican Signed-off-by: necatican Signed-off-by: necatican --- README.md | 2 +- docs/cilium.md | 2 +- .../group_vars/k8s_cluster/k8s-net-cilium.yml | 3 +- roles/download/defaults/main.yml | 2 +- roles/network_plugin/cilium/defaults/main.yml | 11 +- roles/network_plugin/cilium/tasks/check.yml | 9 +- .../templates/cilium-operator/cr.yml.j2 | 74 ++-- .../templates/cilium-operator/deploy.yml.j2 | 119 +++---- .../cilium/templates/cilium/config.yml.j2 | 46 +-- .../cilium/templates/cilium/cr.yml.j2 | 60 ++-- .../cilium/templates/cilium/crb.yml.j2 | 5 - .../cilium/templates/cilium/ds.yml.j2 | 332 +++++++++--------- 12 files changed, 291 insertions(+), 374 deletions(-) diff --git a/README.md b/README.md index 44403da09..ae3e5b36d 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Note: Upstart/SysV init based OS types are not supported. - [cni-plugins](https://github.com/containernetworking/plugins) v1.1.1 - [calico](https://github.com/projectcalico/calico) v3.23.3 - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - - [cilium](https://github.com/cilium/cilium) v1.11.7 + - [cilium](https://github.com/cilium/cilium) v1.12.1 - [flannel](https://github.com/flannel-io/flannel) v0.18.1 - [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.7 - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.5.1 diff --git a/docs/cilium.md b/docs/cilium.md index e9c3e0d2b..e907d53cd 100644 --- a/docs/cilium.md +++ b/docs/cilium.md @@ -56,7 +56,7 @@ cilium_operator_extra_volume_mounts: ## Choose Cilium version ```yml -cilium_version: v1.11.3 +cilium_version: v1.12.1 ``` ## Add variable to config diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml index e82b76138..d6e5bfa61 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml @@ -1,5 +1,5 @@ --- -# cilium_version: "v1.11.7" +# cilium_version: "v1.12.1" # Log-level # cilium_debug: false @@ -118,6 +118,7 @@ # https://docs.cilium.io/en/stable/concepts/networking/masquerading/ # By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded # cilium_ip_masq_agent_enable: false + ### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded # cilium_non_masquerade_cidrs: # - 10.0.0.0/8 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 1889dfbf1..96595d325 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -115,7 +115,7 @@ flannel_cni_version: "v1.1.0" cni_version: "v1.1.1" weave_version: 2.8.1 pod_infra_version: "3.6" -cilium_version: "v1.11.7" +cilium_version: "v1.12.1" kube_ovn_version: "v1.9.7" kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}" kube_router_version: "v1.5.1" diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml index c590637dc..0e624e53c 100644 --- a/roles/network_plugin/cilium/defaults/main.yml +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -1,4 +1,5 @@ --- +cilium_min_version_required: "1.10" # Log-level cilium_debug: false @@ -7,7 +8,7 @@ cilium_enable_ipv4: true cilium_enable_ipv6: false # Cilium agent health port -cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}" +cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879 {%- else -%} 9876 {%- endif -%}" # Identity allocation mode selects how identities are shared between cilium # nodes by setting how they are stored. The options are "crd" or "kvstore". @@ -106,6 +107,7 @@ cilium_wireguard_userspace_fallback: false # https://docs.cilium.io/en/stable/concepts/networking/masquerading/ # By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded cilium_ip_masq_agent_enable: false + ### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded cilium_non_masquerade_cidrs: - 10.0.0.0/8 @@ -201,7 +203,7 @@ cilium_cgroup_host_root: "/run/cilium/cgroupv2" # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. -cilium_bpf_map_dynamic_size_ratio: "{%- if cilium_version | regex_replace('v') is version('1.8', '>=') -%}0.0025{%- else -%}0.0{%- endif -%}" +cilium_bpf_map_dynamic_size_ratio: "0.0025" # -- Enables masquerading of IPv4 traffic leaving the node from endpoints. # Available for Cilium v1.10 and up @@ -240,3 +242,8 @@ cilium_disable_cnp_status_updates: true # Configure how long to wait for the Cilium DaemonSet to be ready again cilium_rolling_restart_wait_retries_count: 30 cilium_rolling_restart_wait_retries_delay_seconds: 10 + +# Cilium changed the default metrics exporter ports in 1.12 +cilium_agent_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9962', '9090') }}" +cilium_operator_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9963', '6942') }}" +cilium_hubble_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9965', '9091') }}" diff --git a/roles/network_plugin/cilium/tasks/check.yml b/roles/network_plugin/cilium/tasks/check.yml index fffa1b53a..c65591f66 100644 --- a/roles/network_plugin/cilium/tasks/check.yml +++ b/roles/network_plugin/cilium/tasks/check.yml @@ -48,13 +48,10 @@ msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'" when: cilium_encryption_enabled -- name: Stop if `cilium_encryption_type` is set to "wireguard" and cilium_version is < v1.10.0 +- name: Stop if cilium_version is < v1.10.0 assert: - that: cilium_version | regex_replace('v') is version('1.10', '>') - msg: "cilium_encryption_type is set to 'wireguard' but cilium_version is < v1.10.0" - when: - - cilium_encryption_enabled - - cilium_encryption_type == "wireguard" + that: cilium_version | regex_replace('v') is version(cilium_min_version_required, '>=') + msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}" # TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled` - name: Set `cilium_encryption_type` to "ipsec" and if `cilium_ipsec_enabled` is true diff --git a/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 index 9f7a71174..8a40a6641 100644 --- a/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 @@ -39,7 +39,14 @@ rules: - get - list - watch -{% if cilium_version | regex_replace('v') is version('1.10', '>=') %} +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -47,22 +54,14 @@ rules: - services/status verbs: - update -{% endif %} - apiGroups: - "" resources: - # to automatically read from k8s and import the node's pod CIDR to cilium's - # etcd so all nodes know how to reach another pod running in in a different - # node. - - nodes # to perform the translation of a CNP that contains `ToGroup` to its endpoints - services - endpoints # to check apiserver connectivity - namespaces -{% if cilium_version | regex_replace('v') is version('1.7', '<') %} - - componentstatuses -{% endif %} verbs: - get - list @@ -72,26 +71,22 @@ rules: resources: - ciliumnetworkpolicies - ciliumnetworkpolicies/status + - ciliumnetworkpolicies/finalizers - ciliumclusterwidenetworkpolicies - ciliumclusterwidenetworkpolicies/status + - ciliumclusterwidenetworkpolicies/finalizers - ciliumendpoints - ciliumendpoints/status -{% if cilium_version | regex_replace('v') is version('1.6', '>=') %} + - ciliumendpoints/finalizers - ciliumnodes - ciliumnodes/status + - ciliumnodes/finalizers - ciliumidentities - ciliumidentities/status -{% endif %} -{% if cilium_version | regex_replace('v') is version('1.9', '>=') %} - - ciliumnetworkpolicies/finalizers - - ciliumclusterwidenetworkpolicies/finalizers - - ciliumendpoints/finalizers - - ciliumnodes/finalizers - ciliumidentities/finalizers - ciliumlocalredirectpolicies - ciliumlocalredirectpolicies/status - ciliumlocalredirectpolicies/finalizers -{% endif %} {% if cilium_version | regex_replace('v') is version('1.11', '>=') %} - ciliumendpointslices {% endif %} @@ -101,12 +96,7 @@ rules: - ciliumenvoyconfigs {% endif %} verbs: - - get - - list - - watch - - create - - update - - delete + - '*' - apiGroups: - apiextensions.k8s.io resources: @@ -117,16 +107,12 @@ rules: - list - update - watch -{% if cilium_version | regex_replace('v') is version('1.8', '>=') %} - # For cilium-operator running in HA mode. - # - # Cilium operator running in HA mode requires the use of ResourceLock for Leader Election - # between mulitple running instances. - # The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less - # common and fewer objects in the cluster watch "all Leases". - # The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release. - # In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure - # that we only authorize access to leases resources in supported K8s versions. +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". - apiGroups: - coordination.k8s.io resources: @@ -135,4 +121,26 @@ rules: - create - get - update +{% if cilium_version | regex_replace('v') is version('1.12', '>=') %} +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumbgploadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumegressnatpolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io {% endif %} diff --git a/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 b/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 index ab8a31926..5a5bd4a92 100644 --- a/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 @@ -2,11 +2,11 @@ apiVersion: apps/v1 kind: Deployment metadata: + name: cilium-operator + namespace: kube-system labels: io.cilium/app: operator name: cilium-operator - name: cilium-operator - namespace: kube-system spec: replicas: {{ cilium_operator_replicas }} selector: @@ -22,29 +22,22 @@ spec: metadata: {% if cilium_enable_prometheus %} annotations: - prometheus.io/port: "6942" + prometheus.io/port: "{{ cilium_operator_scrape_port }}" prometheus.io/scrape: "true" {% endif %} labels: io.cilium/app: operator name: cilium-operator spec: - # In HA mode, cilium-operator pods must not be scheduled on the same - # node as they will clash with each other. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: io.cilium/app - operator: In - values: - - operator - topologyKey: "kubernetes.io/hostname" containers: - - args: - - --debug=$(CILIUM_DEBUG) + - name: cilium-operator + image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - cilium-operator + args: - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) {% if cilium_operator_custom_args is string %} - {{ cilium_operator_custom_args }} {% else %} @@ -52,14 +45,7 @@ spec: - {{ flag }} {% endfor %} {% endif %} - command: - - cilium-operator env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - name: K8S_NODE_NAME valueFrom: fieldRef: @@ -76,45 +62,23 @@ spec: key: debug name: cilium-config optional: true -# We are already mounting the whole ConfigMap as a directory. -# https://github.com/cilium/cilium/pull/10347 -{% if cilium_version | regex_replace('v') is version('1.8', '<') %} - - name: CILIUM_CLUSTER_NAME - valueFrom: - configMapKeyRef: - key: cluster-name - name: cilium-config - optional: true - - name: CILIUM_CLUSTER_ID - valueFrom: - configMapKeyRef: - key: cluster-id - name: cilium-config - optional: true - - name: CILIUM_DISABLE_ENDPOINT_CRD - valueFrom: - configMapKeyRef: - key: disable-endpoint-crd - name: cilium-config - optional: true -{% endif %} - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - key: AWS_ACCESS_KEY_ID name: cilium-aws + key: AWS_ACCESS_KEY_ID optional: true - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - key: AWS_SECRET_ACCESS_KEY name: cilium-aws + key: AWS_SECRET_ACCESS_KEY optional: true - name: AWS_DEFAULT_REGION valueFrom: secretKeyRef: - key: AWS_DEFAULT_REGION name: cilium-aws + key: AWS_DEFAULT_REGION optional: true {% if cilium_kube_proxy_replacement == 'strict' %} - name: KUBERNETES_SERVICE_HOST @@ -122,14 +86,16 @@ spec: - name: KUBERNETES_SERVICE_PORT value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" {% endif %} - image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}" - imagePullPolicy: {{ k8s_image_pull_policy }} - name: cilium-operator {% if cilium_enable_prometheus %} + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace ports: - - containerPort: 6942 - hostPort: 6942 - name: prometheus + - name: prometheus + containerPort: {{ cilium_operator_scrape_port }} + hostPort: {{ cilium_operator_scrape_port }} protocol: TCP {% endif %} livenessProbe: @@ -146,46 +112,55 @@ spec: periodSeconds: 10 timeoutSeconds: 3 volumeMounts: -{% if cilium_identity_allocation_mode == "kvstore" %} - - mountPath: /var/lib/etcd-config - name: etcd-config-path + - name: cilium-config-path + mountPath: /tmp/cilium/config-map readOnly: true - - mountPath: "{{cilium_cert_dir}}" - name: etcd-secrets +{% if cilium_identity_allocation_mode == "kvstore" %} + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + - name: etcd-secrets + mountPath: "{{cilium_cert_dir}}" readOnly: true {% endif %} - - mountPath: /tmp/cilium/config-map - name: cilium-config-path - readOnly: true {% for volume_mount in cilium_operator_extra_volume_mounts %} - {{ volume_mount | to_nice_yaml(indent=2) | indent(14) }} {% endfor %} - dnsPolicy: ClusterFirst - priorityClassName: system-node-critical + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet restartPolicy: Always + priorityClassName: system-node-critical serviceAccount: cilium-operator serviceAccountName: cilium-operator - hostNetwork: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + io.cilium/app: operator tolerations: - operator: Exists volumes: + - name: cilium-config-path + configMap: + name: cilium-config {% if cilium_identity_allocation_mode == "kvstore" %} # To read the etcd config stored in config maps - - configMap: + - name: etcd-config-path + configMap: + name: cilium-config defaultMode: 420 items: - key: etcd-config path: etcd.config - name: cilium-config - name: etcd-config-path # To read the k8s etcd secrets in case the user might want to use TLS - name: etcd-secrets hostPath: path: "{{cilium_cert_dir}}" {% endif %} - - configMap: - name: cilium-config - name: cilium-config-path {% for volume in cilium_operator_extra_volumes %} - {{ volume | to_nice_yaml(indent=2) | indent(10) }} {% endfor %} diff --git a/roles/network_plugin/cilium/templates/cilium/config.yml.j2 b/roles/network_plugin/cilium/templates/cilium/config.yml.j2 index 75232991f..fadc87b64 100644 --- a/roles/network_plugin/cilium/templates/cilium/config.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium/config.yml.j2 @@ -41,8 +41,8 @@ data: # NOTE that this will open the port on ALL nodes where Cilium pods are # scheduled. {% if cilium_enable_prometheus %} - prometheus-serve-addr: ":9090" - operator-prometheus-serve-addr: ":6942" + prometheus-serve-addr: ":{{ cilium_agent_scrape_port }}" + operator-prometheus-serve-addr: ":{{ cilium_operator_scrape_port }}" enable-metrics: "true" {% endif %} @@ -127,31 +127,6 @@ data: cluster-id: "{{ cilium_cluster_id }}" {% endif %} -# `tofqdns-enable-poller` is deprecated in 1.8, removed in 1.9 -# https://github.com/cilium/cilium/issues/8604 -{% if cilium_version | regex_replace('v') is version('1.9', '<') %} - # DNS Polling periodically issues a DNS lookup for each `matchName` from - # cilium-agent. The result is used to regenerate endpoint policy. - # DNS lookups are repeated with an interval of 5 seconds, and are made for - # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP - # data is used instead. An IP change will trigger a regeneration of the Cilium - # policy for each endpoint and increment the per cilium-agent policy - # repository revision. - # - # This option is disabled by default starting from version 1.4.x in favor - # of a more powerful DNS proxy-based implementation, see [0] for details. - # Enable this option if you want to use FQDN policies but do not want to use - # the DNS proxy. - # - # To ease upgrade, users may opt to set this option to "true". - # Otherwise please refer to the Upgrade Guide [1] which explains how to - # prepare policy rules for upgrade. - # - # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based - # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action - tofqdns-enable-poller: "{{cilium_tofqdns_enable_poller}}" -{% endif %} - # `wait-bpf-mount` is removed after v1.10.4 # https://github.com/cilium/cilium/commit/d2217045cb3726a7f823174e086913b69b8090da {% if cilium_version | regex_replace('v') is version('1.10.4', '<') %} @@ -159,14 +134,6 @@ data: wait-bpf-mount: "false" {% endif %} -# `enable-legacy-services` is deprecated in 1.6, removed in 1.9 -# https://github.com/cilium/cilium/pull/10255 -{% if cilium_version | regex_replace('v') is version('1.9', '<') %} - # Enable legacy services (prior v1.5) to prevent from terminating existing - # connections with services when upgrading Cilium from < v1.5 to v1.5. - enable-legacy-services: "{{cilium_enable_legacy_services}}" -{% endif %} - kube-proxy-replacement: "{{ cilium_kube_proxy_replacement }}" # `native-routing-cidr` is deprecated in 1.10, removed in 1.12. @@ -191,7 +158,7 @@ data: {% if cilium_enable_hubble %} enable-hubble: "true" {% if cilium_enable_hubble_metrics %} - hubble-metrics-server: ":9091" + hubble-metrics-server: ":{{ cilium_hubble_scrape_port }}" hubble-metrics: {% for hubble_metrics_cycle in cilium_hubble_metrics %} {{ hubble_metrics_cycle }} @@ -228,9 +195,7 @@ data: {% endif %} # IPAM settings -{% if cilium_version | regex_replace('v') is version('1.9', '>=') %} ipam: "{{ cilium_ipam_mode }}" -{% endif %} agent-health-port: "{{ cilium_agent_health_port }}" @@ -240,10 +205,8 @@ data: bpf-map-dynamic-size-ratio: "{{ cilium_bpf_map_dynamic_size_ratio }}" -{% if cilium_version | regex_replace('v') is version('1.10', '>=') %} enable-ipv4-masquerade: "{{ cilium_enable_ipv4_masquerade }}" enable-ipv6-masquerade: "{{ cilium_enable_ipv6_masquerade }}" -{% endif %} enable-bpf-masquerade: "{{ cilium_enable_bpf_masquerade }}" @@ -258,7 +221,6 @@ data: enable-bpf-clock-probe: "{{ cilium_enable_bpf_clock_probe }}" disable-cnp-status-updates: "{{ cilium_disable_cnp_status_updates }}" - {% if cilium_ip_masq_agent_enable %} --- apiVersion: v1 @@ -274,4 +236,4 @@ data: {% endfor %} masqLinkLocal: {{ cilium_masq_link_local|bool }} resyncInterval: "{{ cilium_ip_masq_resync_interval }}" -{% endif %} +{% endif %} \ No newline at end of file diff --git a/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 index 02d8e6bf7..a16211c17 100644 --- a/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 @@ -7,9 +7,6 @@ rules: - apiGroups: - networking.k8s.io resources: -{% if cilium_version | regex_replace('v') is version('1.7', '<') %} - - ingresses -{% endif %} - networkpolicies verbs: - get @@ -28,34 +25,25 @@ rules: resources: - namespaces - services - - nodes + - pods - endpoints -{% if cilium_version | regex_replace('v') is version('1.7', '<') %} - - componentstatuses -{% endif %} + - nodes verbs: - get - list - watch -{% if cilium_version | regex_replace('v') is version('1.7', '<') %} -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - create - - get - - list - - watch -{% endif %} -{% if cilium_version | regex_replace('v') is version('1.7', '>') %} +{% if cilium_version | regex_replace('v') is version('1.12', '<') %} - apiGroups: - "" resources: + - pods - pods/finalizers verbs: + - get + - list + - watch - update -{% endif %} + - delete - apiGroups: - "" resources: @@ -66,6 +54,7 @@ rules: - list - watch - update +{% endif %} - apiGroups: - "" resources: @@ -78,47 +67,45 @@ rules: resources: - customresourcedefinitions verbs: + # Deprecated for removal in v1.10 - create - - get - list - watch - update + + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get - apiGroups: - cilium.io resources: - ciliumnetworkpolicies - ciliumnetworkpolicies/status -{% if cilium_version | regex_replace('v') is version('1.7', '>=') %} - ciliumclusterwidenetworkpolicies - ciliumclusterwidenetworkpolicies/status -{% endif %} - ciliumendpoints - ciliumendpoints/status -{% if cilium_version | regex_replace('v') is version('1.6', '>=') %} - ciliumnodes - ciliumnodes/status - ciliumidentities - - ciliumidentities/status -{% endif %} -{% if cilium_version | regex_replace('v') is version('1.9', '>=') %} - - ciliumnetworkpolicies/finalizers - - ciliumclusterwidenetworkpolicies/finalizers - - ciliumendpoints/finalizers - - ciliumnodes/finalizers - - ciliumidentities/finalizers - ciliumlocalredirectpolicies - ciliumlocalredirectpolicies/status - - ciliumlocalredirectpolicies/finalizers -{% endif %} -{% if cilium_version | regex_replace('v') is version('1.10', '>=') %} - ciliumegressnatpolicies -{% endif %} {% if cilium_version | regex_replace('v') is version('1.11', '>=') %} - ciliumendpointslices {% endif %} {% if cilium_version | regex_replace('v') is version('1.12', '>=') %} - ciliumbgploadbalancerippools - ciliumbgppeeringpolicies +{% endif %} +{% if cilium_version | regex_replace('v') is version('1.11.5', '<') %} + - ciliumnetworkpolicies/finalizers + - ciliumclusterwidenetworkpolicies/finalizers + - ciliumendpoints/finalizers + - ciliumnodes/finalizers + - ciliumidentities/finalizers + - ciliumlocalredirectpolicies/finalizers {% endif %} verbs: - '*' @@ -128,6 +115,7 @@ rules: resources: - ciliumclusterwideenvoyconfigs - ciliumenvoyconfigs + - ciliumegressgatewaypolicies verbs: - list - watch diff --git a/roles/network_plugin/cilium/templates/cilium/crb.yml.j2 b/roles/network_plugin/cilium/templates/cilium/crb.yml.j2 index f7516d707..d23897fa0 100644 --- a/roles/network_plugin/cilium/templates/cilium/crb.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium/crb.yml.j2 @@ -11,8 +11,3 @@ subjects: - kind: ServiceAccount name: cilium namespace: kube-system -{% if cilium_version | regex_replace('v') is version('1.9', '<') %} -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes -{% endif %} diff --git a/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 index d5ceaa676..2ce442062 100644 --- a/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 @@ -2,37 +2,37 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - labels: - k8s-app: cilium name: cilium namespace: kube-system + labels: + k8s-app: cilium spec: selector: matchLabels: k8s-app: cilium + updateStrategy: + rollingUpdate: + # Specifies the maximum number of Pods that can be unavailable during the update process. + maxUnavailable: 2 + type: RollingUpdate template: metadata: annotations: {% if cilium_enable_prometheus %} - prometheus.io/port: "9090" + prometheus.io/port: "{{ cilium_agent_scrape_port }}" prometheus.io/scrape: "true" {% endif %} scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]' labels: k8s-app: cilium spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: k8s-app - operator: In - values: - - cilium - topologyKey: kubernetes.io/hostname containers: - - args: + - name: cilium-agent + image: "{{cilium_image_repo}}:{{cilium_image_tag}}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - cilium-agent + args: - --config-dir=/tmp/cilium/config-map {% if cilium_mtu != "" %} - --mtu={{ cilium_mtu }} @@ -44,54 +44,6 @@ spec: - {{ flag }} {% endfor %} {% endif %} - command: - - cilium-agent - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_CLUSTERMESH_CONFIG - value: /var/lib/cilium/clustermesh/ -{% if cilium_kube_proxy_replacement == 'strict' %} - - name: KUBERNETES_SERVICE_HOST - value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" - - name: KUBERNETES_SERVICE_PORT - value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" -{% endif %} -{% for env_var in cilium_agent_extra_env_vars %} - - {{ env_var | to_nice_yaml(indent=2) | indent(10) }} -{% endfor %} - image: "{{cilium_image_repo}}:{{cilium_image_tag}}" - imagePullPolicy: {{ k8s_image_pull_policy }} - resources: - limits: - cpu: {{ cilium_cpu_limit }} - memory: {{ cilium_memory_limit }} - requests: - cpu: {{ cilium_cpu_requests }} - memory: {{ cilium_memory_requests }} - lifecycle: - postStart: - exec: - command: - - "/cni-install.sh" -{% if cilium_version | regex_replace('v') is version('1.10', '>=') %} - - "--cni-exclusive={{ cilium_cni_exclusive }}" -{% endif %} -{% if cilium_version | regex_replace('v') is version('1.12', '>=') %} - - "--log-file={{ cilium_cni_log_file }}" -{% endif %} - preStop: - exec: - command: - - /cni-uninstall.sh startupProbe: httpGet: host: '127.0.0.1' @@ -131,80 +83,109 @@ spec: successThreshold: 1 failureThreshold: 3 timeoutSeconds: 5 - name: cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ +{% if cilium_kube_proxy_replacement == 'strict' %} + - name: KUBERNETES_SERVICE_HOST + value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" + - name: KUBERNETES_SERVICE_PORT + value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" +{% endif %} +{% for env_var in cilium_agent_extra_env_vars %} + - {{ env_var | to_nice_yaml(indent=2) | indent(10) }} +{% endfor %} + lifecycle: + postStart: + exec: + command: + - "/cni-install.sh" + - "--cni-exclusive={{ cilium_cni_exclusive | string | lower }}" +{% if cilium_version | regex_replace('v') is version('1.12', '>=') %} + - "--enable-debug={{ cilium_debug | string | lower }}" + - "--log-file={{ cilium_cni_log_file }}" +{% endif %} + preStop: + exec: + command: + - /cni-uninstall.sh + resources: + limits: + cpu: {{ cilium_cpu_limit }} + memory: {{ cilium_memory_limit }} + requests: + cpu: {{ cilium_cpu_requests }} + memory: {{ cilium_memory_requests }} {% if cilium_enable_prometheus or cilium_enable_hubble_metrics %} ports: {% endif %} {% if cilium_enable_prometheus %} - - containerPort: 9090 - hostPort: 9090 - name: prometheus + - name: prometheus + containerPort: {{ cilium_agent_scrape_port }} + hostPort: {{ cilium_agent_scrape_port }} protocol: TCP {% endif %} {% if cilium_enable_hubble_metrics %} - - containerPort: 9091 - hostPort: 9091 - name: hubble-metrics + - name: hubble-metrics + containerPort: {{ cilium_hubble_scrape_port }} + hostPort: {{ cilium_hubble_scrape_port }} protocol: TCP {% endif %} securityContext: privileged: true volumeMounts: - - mountPath: /sys/fs/bpf - name: bpf-maps + - name: bpf-maps + mountPath: /sys/fs/bpf mountPropagation: Bidirectional - - mountPath: /var/run/cilium - name: cilium-run - - mountPath: /host/opt/cni/bin - name: cni-path - - mountPath: /host/etc/cni/net.d - name: etc-cni-netd -# pkg/workloads was depreca, removed in 1.7 -# https://github.com/cilium/cilium/pull/9447 -{% if cilium_version | regex_replace('v') is version('1.7', '<') %} -{% if container_manager == 'docker' %} - - mountPath: /var/run/docker.sock - name: docker-socket - readOnly: true -{% else %} - - name: "{{ container_manager }}-socket" - mountPath: {{ cri_socket }} - readOnly: true -{% endif %} -{% endif %} + - name: cilium-run + mountPath: /var/run/cilium + - name: cni-path + mountPath: /host/opt/cni/bin + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d {% if cilium_identity_allocation_mode == "kvstore" %} - - mountPath: /var/lib/etcd-config - name: etcd-config-path + - name: etcd-config-path + mountPath: /var/lib/etcd-config readOnly: true - - mountPath: "{{cilium_cert_dir}}" - name: etcd-secrets + - name: etcd-secrets + mountPath: "{{cilium_cert_dir}}" readOnly: true {% endif %} - - mountPath: /var/lib/cilium/clustermesh - name: clustermesh-secrets + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh readOnly: true - - mountPath: /tmp/cilium/config-map - name: cilium-config-path + - name: cilium-config-path + mountPath: /tmp/cilium/config-map readOnly: true -{% if cilium_ip_masq_agent_enable %} +{% if cilium_ip_masq_agent_enable %}\ - name: ip-masq-agent mountPath: /etc/config readOnly: true {% endif %} # Needed to be able to load kernel modules - - mountPath: /lib/modules - name: lib-modules + - name: lib-modules + mountPath: /lib/modules readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock + - name: xtables-lock + mountPath: /run/xtables.lock {% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %} - - mountPath: /etc/ipsec - name: cilium-ipsec-secrets + - name: cilium-ipsec-secrets + mountPath: /etc/ipsec readOnly: true {% endif %} {% if cilium_hubble_install %} - - mountPath: /var/lib/cilium/tls/hubble - name: hubble-tls + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble readOnly: true {% endif %} {% for volume_mount in cilium_agent_extra_volume_mounts %} @@ -245,22 +226,50 @@ spec: securityContext: privileged: true {% endif %} - - command: +{% if cilium_version | regex_replace('v') is version('1.11.7', '>=') %} + - name: apply-sysctl-overwrites + image: "{{cilium_image_repo}}:{{cilium_image_tag}}" + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + securityContext: + privileged: true +{% endif %} + - name: clean-cilium-state + image: "{{cilium_image_repo}}:{{cilium_image_tag}}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: - /init-container.sh env: - name: CILIUM_ALL_STATE valueFrom: configMapKeyRef: - key: clean-cilium-state name: cilium-config + key: clean-cilium-state optional: true - # CLEAN_CILIUM_BPF_STATE is deprecated in 1.6. - # https://github.com/cilium/cilium/pull/7478 - - name: "{{ cilium_version | regex_replace('v') is version('1.6', '<')| ternary('CLEAN_CILIUM_BPF_STATE','CILIUM_BPF_STATE') }}" + - name: CILIUM_BPF_STATE valueFrom: configMapKeyRef: - key: clean-cilium-bpf-state name: cilium-config + key: clean-cilium-bpf-state optional: true # Removed in 1.11 and up. # https://github.com/cilium/cilium/commit/f7a3f59fd74983c600bfce9cac364b76d20849d9 @@ -278,32 +287,25 @@ spec: - name: KUBERNETES_SERVICE_PORT value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" {% endif %} -{% if cilium_version | regex_replace('v') is version('1.9', '<') %} - image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}" -{% else %} - image: "{{cilium_image_repo}}:{{cilium_image_tag}}" -{% endif %} - imagePullPolicy: {{ k8s_image_pull_policy }} - name: clean-cilium-state securityContext: privileged: true volumeMounts: - - mountPath: /sys/fs/bpf - name: bpf-maps + - name: bpf-maps + mountPath: /sys/fs/bpf {% if cilium_version | regex_replace('v') is version('1.11', '>=') %} # Required to mount cgroup filesystem from the host to cilium agent pod - name: cilium-cgroup mountPath: {{ cilium_cgroup_host_root }} mountPropagation: HostToContainer {% endif %} - - mountPath: /var/run/cilium - name: cilium-run + - name: cilium-run + mountPath: /var/run/cilium resources: requests: cpu: 100m memory: 100Mi - priorityClassName: system-node-critical restartPolicy: Always + priorityClassName: system-node-critical serviceAccount: cilium serviceAccountName: cilium terminationGracePeriodSeconds: 1 @@ -312,36 +314,26 @@ spec: {% if cilium_identity_allocation_mode == "kvstore" %} dnsPolicy: ClusterFirstWithHostNet {% endif %} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium tolerations: - operator: Exists volumes: # To keep state between restarts / upgrades - - hostPath: + - name: cilium-run + hostPath: path: /var/run/cilium type: DirectoryOrCreate - name: cilium-run # To keep state between restarts / upgrades for bpf maps - - hostPath: + - name: bpf-maps + hostPath: path: /sys/fs/bpf type: DirectoryOrCreate - name: bpf-maps -# pkg/workloads was deprecated in 1.6, removed in 1.7 -# https://github.com/cilium/cilium/pull/9447 -{% if cilium_version | regex_replace('v') is version('1.7', '<') %} -{% if container_manager == 'docker' %} - # To read docker events from the node - - hostPath: - path: /var/run/docker.sock - type: Socket - name: docker-socket -{% else %} - # To read crio events from the node - - hostPath: - path: {{ cri_socket }} - type: Socket - name: {{ container_manager }}-socket -{% endif %} -{% endif %} {% if cilium_version | regex_replace('v') is version('1.11', '>=') %} # To mount cgroup2 filesystem on the host - name: hostproc @@ -355,34 +347,34 @@ spec: type: DirectoryOrCreate {% endif %} # To install cilium cni plugin in the host - - hostPath: + - name: cni-path + hostPath: path: /opt/cni/bin type: DirectoryOrCreate - name: cni-path # To install cilium cni configuration in the host - - hostPath: + - name: etc-cni-netd + hostPath: path: /etc/cni/net.d type: DirectoryOrCreate - name: etc-cni-netd # To be able to load kernel modules - - hostPath: + - name: lib-modules + hostPath: path: /lib/modules - name: lib-modules # To access iptables concurrently with other processes (e.g. kube-proxy) - - hostPath: + - name: xtables-lock + hostPath: path: /run/xtables.lock type: FileOrCreate - name: xtables-lock {% if cilium_identity_allocation_mode == "kvstore" %} # To read the etcd config stored in config maps - - configMap: + - name: etcd-config-path + configMap: + name: cilium-config # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 items: - key: etcd-config path: etcd.config - name: cilium-config - name: etcd-config-path # To read the k8s etcd secrets in case the user might want to use TLS - name: etcd-secrets hostPath: @@ -391,21 +383,22 @@ spec: # To read the clustermesh configuration - name: clustermesh-secrets secret: + secretName: cilium-clustermesh # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 optional: true - secretName: cilium-clustermesh # To read the configuration from the config map - - configMap: + - name: cilium-config-path + configMap: name: cilium-config - name: cilium-config-path {% if cilium_ip_masq_agent_enable %} - - configMap: + - name: ip-masq-agent + configMap: name: ip-masq-agent + optional: true items: - key: config path: ip-masq-agent - name: ip-masq-agent {% endif %} {% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %} - name: cilium-ipsec-secrets @@ -420,21 +413,12 @@ spec: sources: - secret: name: hubble-server-certs + optional: true items: + - key: ca.crt + path: client-ca.crt - key: tls.crt path: server.crt - key: tls.key path: server.key - optional: true - - configMap: - name: hubble-ca-cert - items: - - key: ca.crt - path: client-ca.crt - optional: true {% endif %} - updateStrategy: - rollingUpdate: - # Specifies the maximum number of Pods that can be unavailable during the update process. - maxUnavailable: 2 - type: RollingUpdate