kubernetes-sigs
/
kubespray
mirror of https://github.com/kubernetes-sigs/kubespray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cilium 1.12 Upgrade (#9225) 

* Drop support for Cilium < 1.10

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* Synchronize Cilium templates for 1.11.7

Signed-off-by: necatican <contact@necatican.com>

* Set Cilium v1.12.1 as the default version

Signed-off-by: necatican <contact@necatican.com>

Signed-off-by: necatican <necaticanyildirim@gmail.com>
Signed-off-by: necatican <contact@necatican.com>
pull/9296/head
Necatican Yıldırım 2022-09-19 12:14:31 +03:00 committed by GitHub
parent 680293e79c
commit 7da3dbcb39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 291 additions and 374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -144,7 +144,7 @@ Note: Upstart/SysV init based OS types are not supported.
- [cni-plugins](https://github.com/containernetworking/plugins) v1.1.1
- [calico](https://github.com/projectcalico/calico) v3.23.3
- [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
- [cilium](https://github.com/cilium/cilium) v1.11.7
- [cilium](https://github.com/cilium/cilium) v1.12.1
- [flannel](https://github.com/flannel-io/flannel) v0.18.1
- [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.7
- [kube-router](https://github.com/cloudnativelabs/kube-router) v1.5.1

2
docs/cilium.md
View File

@ -56,7 +56,7 @@ cilium_operator_extra_volume_mounts:
## Choose Cilium version
```yml
cilium_version: v1.11.3
cilium_version: v1.12.1
```
## Add variable to config

3
inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml
View File

@ -1,5 +1,5 @@
---
# cilium_version: "v1.11.7"
# cilium_version: "v1.12.1"
# Log-level
# cilium_debug: false
@ -118,6 +118,7 @@
# https://docs.cilium.io/en/stable/concepts/networking/masquerading/
# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
# cilium_ip_masq_agent_enable: false
### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
# cilium_non_masquerade_cidrs:
# - 10.0.0.0/8

2
roles/download/defaults/main.yml
View File

@ -115,7 +115,7 @@ flannel_cni_version: "v1.1.0"
cni_version: "v1.1.1"
weave_version: 2.8.1
pod_infra_version: "3.6"
cilium_version: "v1.11.7"
cilium_version: "v1.12.1"
kube_ovn_version: "v1.9.7"
kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}"
kube_router_version: "v1.5.1"

11
roles/network_plugin/cilium/defaults/main.yml
View File

@ -1,4 +1,5 @@
---
cilium_min_version_required: "1.10"
# Log-level
cilium_debug: false
@ -7,7 +8,7 @@ cilium_enable_ipv4: true
cilium_enable_ipv6: false
# Cilium agent health port
cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}"
cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879 {%- else -%} 9876 {%- endif -%}"
# Identity allocation mode selects how identities are shared between cilium
# nodes by setting how they are stored. The options are "crd" or "kvstore".
@ -106,6 +107,7 @@ cilium_wireguard_userspace_fallback: false
# https://docs.cilium.io/en/stable/concepts/networking/masquerading/
# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
cilium_ip_masq_agent_enable: false
### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
cilium_non_masquerade_cidrs:
- 10.0.0.0/8
@ -201,7 +203,7 @@ cilium_cgroup_host_root: "/run/cilium/cgroupv2"
# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
cilium_bpf_map_dynamic_size_ratio: "{%- if cilium_version | regex_replace('v') is version('1.8', '>=') -%}0.0025{%- else -%}0.0{%- endif -%}"
cilium_bpf_map_dynamic_size_ratio: "0.0025"
# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
# Available for Cilium v1.10 and up
@ -240,3 +242,8 @@ cilium_disable_cnp_status_updates: true
# Configure how long to wait for the Cilium DaemonSet to be ready again
cilium_rolling_restart_wait_retries_count: 30
cilium_rolling_restart_wait_retries_delay_seconds: 10
# Cilium changed the default metrics exporter ports in 1.12
cilium_agent_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9962', '9090') }}"
cilium_operator_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9963', '6942') }}"
cilium_hubble_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9965', '9091') }}"

9
roles/network_plugin/cilium/tasks/check.yml
View File

@ -48,13 +48,10 @@
msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'"
when: cilium_encryption_enabled
- name: Stop if `cilium_encryption_type` is set to "wireguard" and cilium_version is < v1.10.0
- name: Stop if cilium_version is < v1.10.0
assert:
that: cilium_version | regex_replace('v') is version('1.10', '>')
msg: "cilium_encryption_type is set to 'wireguard' but cilium_version is < v1.10.0"
when:
- cilium_encryption_enabled
- cilium_encryption_type == "wireguard"
that: cilium_version | regex_replace('v') is version(cilium_min_version_required, '>=')
msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}"
# TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled`
- name: Set `cilium_encryption_type` to "ipsec" and if `cilium_ipsec_enabled` is true

74
roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
View File

@ -39,7 +39,14 @@ rules:
- get
- list
- watch
{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
@ -47,22 +54,14 @@ rules:
- services/status
verbs:
- update
{% endif %}
- apiGroups:
- ""
resources:
# to automatically read from k8s and import the node's pod CIDR to cilium's
# etcd so all nodes know how to reach another pod running in in a different
# node.
- nodes
# to perform the translation of a CNP that contains `ToGroup` to its endpoints
- services
- endpoints
# to check apiserver connectivity
- namespaces
{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
- componentstatuses
{% endif %}
verbs:
- get
- list
@ -72,26 +71,22 @@ rules:
resources:
- ciliumnetworkpolicies
- ciliumnetworkpolicies/status
- ciliumnetworkpolicies/finalizers
- ciliumclusterwidenetworkpolicies
- ciliumclusterwidenetworkpolicies/status
- ciliumclusterwidenetworkpolicies/finalizers
- ciliumendpoints
- ciliumendpoints/status
{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
- ciliumendpoints/finalizers
- ciliumnodes
- ciliumnodes/status
- ciliumnodes/finalizers
- ciliumidentities
- ciliumidentities/status
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
- ciliumnetworkpolicies/finalizers
- ciliumclusterwidenetworkpolicies/finalizers
- ciliumendpoints/finalizers
- ciliumnodes/finalizers
- ciliumidentities/finalizers
- ciliumlocalredirectpolicies
- ciliumlocalredirectpolicies/status
- ciliumlocalredirectpolicies/finalizers
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
- ciliumendpointslices
{% endif %}
@ -101,12 +96,7 @@ rules:
- ciliumenvoyconfigs
{% endif %}
verbs:
- get
- list
- watch
- create
- update
- delete
- '*'
- apiGroups:
- apiextensions.k8s.io
resources:
@ -117,16 +107,12 @@ rules:
- list
- update
- watch
{% if cilium_version | regex_replace('v') is version('1.8', '>=') %}
# For cilium-operator running in HA mode.
#
# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
# between mulitple running instances.
# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
# common and fewer objects in the cluster watch "all Leases".
# The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release.
# In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure
# that we only authorize access to leases resources in supported K8s versions.
# For cilium-operator running in HA mode.
#
# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
# between multiple running instances.
# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
# common and fewer objects in the cluster watch "all Leases".
- apiGroups:
- coordination.k8s.io
resources:
@ -135,4 +121,26 @@ rules:
- create
- get
- update
{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- update
resourceNames:
- ciliumbgploadbalancerippools.cilium.io
- ciliumbgppeeringpolicies.cilium.io
- ciliumclusterwideenvoyconfigs.cilium.io
- ciliumclusterwidenetworkpolicies.cilium.io
- ciliumegressgatewaypolicies.cilium.io
- ciliumegressnatpolicies.cilium.io
- ciliumendpoints.cilium.io
- ciliumendpointslices.cilium.io
- ciliumenvoyconfigs.cilium.io
- ciliumexternalworkloads.cilium.io
- ciliumidentities.cilium.io
- ciliumlocalredirectpolicies.cilium.io
- ciliumnetworkpolicies.cilium.io
- ciliumnodes.cilium.io
{% endif %}

119
roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2
View File

@ -2,11 +2,11 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: cilium-operator
namespace: kube-system
labels:
io.cilium/app: operator
name: cilium-operator
name: cilium-operator
namespace: kube-system
spec:
replicas: {{ cilium_operator_replicas }}
selector:
@ -22,29 +22,22 @@ spec:
metadata:
{% if cilium_enable_prometheus %}
annotations:
prometheus.io/port: "6942"
prometheus.io/port: "{{ cilium_operator_scrape_port }}"
prometheus.io/scrape: "true"
{% endif %}
labels:
io.cilium/app: operator
name: cilium-operator
spec:
# In HA mode, cilium-operator pods must not be scheduled on the same
# node as they will clash with each other.
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: io.cilium/app
operator: In
values:
- operator
topologyKey: "kubernetes.io/hostname"
containers:
- args:
- --debug=$(CILIUM_DEBUG)
- name: cilium-operator
image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
imagePullPolicy: {{ k8s_image_pull_policy }}
command:
- cilium-operator
args:
- --config-dir=/tmp/cilium/config-map
- --debug=$(CILIUM_DEBUG)
{% if cilium_operator_custom_args is string %}
- {{ cilium_operator_custom_args }}
{% else %}
@ -52,14 +45,7 @@ spec:
- {{ flag }}
{% endfor %}
{% endif %}
command:
- cilium-operator
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
@ -76,45 +62,23 @@ spec:
key: debug
name: cilium-config
optional: true
# We are already mounting the whole ConfigMap as a directory.
# https://github.com/cilium/cilium/pull/10347
{% if cilium_version | regex_replace('v') is version('1.8', '<') %}
- name: CILIUM_CLUSTER_NAME
valueFrom:
configMapKeyRef:
key: cluster-name
name: cilium-config
optional: true
- name: CILIUM_CLUSTER_ID
valueFrom:
configMapKeyRef:
key: cluster-id
name: cilium-config
optional: true
- name: CILIUM_DISABLE_ENDPOINT_CRD
valueFrom:
configMapKeyRef:
key: disable-endpoint-crd
name: cilium-config
optional: true
{% endif %}
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
key: AWS_ACCESS_KEY_ID
name: cilium-aws
key: AWS_ACCESS_KEY_ID
optional: true
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
key: AWS_SECRET_ACCESS_KEY
name: cilium-aws
key: AWS_SECRET_ACCESS_KEY
optional: true
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
key: AWS_DEFAULT_REGION
name: cilium-aws
key: AWS_DEFAULT_REGION
optional: true
{% if cilium_kube_proxy_replacement == 'strict' %}
- name: KUBERNETES_SERVICE_HOST
@ -122,14 +86,16 @@ spec:
- name: KUBERNETES_SERVICE_PORT
value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
{% endif %}
image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
imagePullPolicy: {{ k8s_image_pull_policy }}
name: cilium-operator
{% if cilium_enable_prometheus %}
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
ports:
- containerPort: 6942
hostPort: 6942
name: prometheus
- name: prometheus
containerPort: {{ cilium_operator_scrape_port }}
hostPort: {{ cilium_operator_scrape_port }}
protocol: TCP
{% endif %}
livenessProbe:
@ -146,46 +112,55 @@ spec:
periodSeconds: 10
timeoutSeconds: 3
volumeMounts:
{% if cilium_identity_allocation_mode == "kvstore" %}
- mountPath: /var/lib/etcd-config
name: etcd-config-path
- name: cilium-config-path
mountPath: /tmp/cilium/config-map
readOnly: true
- mountPath: "{{cilium_cert_dir}}"
name: etcd-secrets
{% if cilium_identity_allocation_mode == "kvstore" %}
- name: etcd-config-path
mountPath: /var/lib/etcd-config
readOnly: true
- name: etcd-secrets
mountPath: "{{cilium_cert_dir}}"
readOnly: true
{% endif %}
- mountPath: /tmp/cilium/config-map
name: cilium-config-path
readOnly: true
{% for volume_mount in cilium_operator_extra_volume_mounts %}
- {{ volume_mount | to_nice_yaml(indent=2) | indent(14) }}
{% endfor %}
dnsPolicy: ClusterFirst
priorityClassName: system-node-critical
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
restartPolicy: Always
priorityClassName: system-node-critical
serviceAccount: cilium-operator
serviceAccountName: cilium-operator
hostNetwork: true
# In HA mode, cilium-operator pods must not be scheduled on the same
# node as they will clash with each other.
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
io.cilium/app: operator
tolerations:
- operator: Exists
volumes:
- name: cilium-config-path
configMap:
name: cilium-config
{% if cilium_identity_allocation_mode == "kvstore" %}
# To read the etcd config stored in config maps
- configMap:
- name: etcd-config-path
configMap:
name: cilium-config
defaultMode: 420
items:
- key: etcd-config
path: etcd.config
name: cilium-config
name: etcd-config-path
# To read the k8s etcd secrets in case the user might want to use TLS
- name: etcd-secrets
hostPath:
path: "{{cilium_cert_dir}}"
{% endif %}
- configMap:
name: cilium-config
name: cilium-config-path
{% for volume in cilium_operator_extra_volumes %}
- {{ volume | to_nice_yaml(indent=2) | indent(10) }}
{% endfor %}

46
roles/network_plugin/cilium/templates/cilium/config.yml.j2
View File

@ -41,8 +41,8 @@ data:
# NOTE that this will open the port on ALL nodes where Cilium pods are
# scheduled.
{% if cilium_enable_prometheus %}
prometheus-serve-addr: ":9090"
operator-prometheus-serve-addr: ":6942"
prometheus-serve-addr: ":{{ cilium_agent_scrape_port }}"
operator-prometheus-serve-addr: ":{{ cilium_operator_scrape_port }}"
enable-metrics: "true"
{% endif %}
@ -127,31 +127,6 @@ data:
cluster-id: "{{ cilium_cluster_id }}"
{% endif %}
# `tofqdns-enable-poller` is deprecated in 1.8, removed in 1.9
# https://github.com/cilium/cilium/issues/8604
{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
# DNS Polling periodically issues a DNS lookup for each `matchName` from
# cilium-agent. The result is used to regenerate endpoint policy.
# DNS lookups are repeated with an interval of 5 seconds, and are made for
# A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP
# data is used instead. An IP change will trigger a regeneration of the Cilium
# policy for each endpoint and increment the per cilium-agent policy
# repository revision.
#
# This option is disabled by default starting from version 1.4.x in favor
# of a more powerful DNS proxy-based implementation, see [0] for details.
# Enable this option if you want to use FQDN policies but do not want to use
# the DNS proxy.
#
# To ease upgrade, users may opt to set this option to "true".
# Otherwise please refer to the Upgrade Guide [1] which explains how to
# prepare policy rules for upgrade.
#
# [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
# [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
tofqdns-enable-poller: "{{cilium_tofqdns_enable_poller}}"
{% endif %}
# `wait-bpf-mount` is removed after v1.10.4
# https://github.com/cilium/cilium/commit/d2217045cb3726a7f823174e086913b69b8090da
{% if cilium_version | regex_replace('v') is version('1.10.4', '<') %}
@ -159,14 +134,6 @@ data:
wait-bpf-mount: "false"
{% endif %}
# `enable-legacy-services` is deprecated in 1.6, removed in 1.9
# https://github.com/cilium/cilium/pull/10255
{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
# Enable legacy services (prior v1.5) to prevent from terminating existing
# connections with services when upgrading Cilium from < v1.5 to v1.5.
enable-legacy-services: "{{cilium_enable_legacy_services}}"
{% endif %}
kube-proxy-replacement: "{{ cilium_kube_proxy_replacement }}"
# `native-routing-cidr` is deprecated in 1.10, removed in 1.12.
@ -191,7 +158,7 @@ data:
{% if cilium_enable_hubble %}
enable-hubble: "true"
{% if cilium_enable_hubble_metrics %}
hubble-metrics-server: ":9091"
hubble-metrics-server: ":{{ cilium_hubble_scrape_port }}"
hubble-metrics:
{% for hubble_metrics_cycle in cilium_hubble_metrics %}
{{ hubble_metrics_cycle }}
@ -228,9 +195,7 @@ data:
{% endif %}
# IPAM settings
{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
ipam: "{{ cilium_ipam_mode }}"
{% endif %}
agent-health-port: "{{ cilium_agent_health_port }}"
@ -240,10 +205,8 @@ data:
bpf-map-dynamic-size-ratio: "{{ cilium_bpf_map_dynamic_size_ratio }}"
{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
enable-ipv4-masquerade: "{{ cilium_enable_ipv4_masquerade }}"
enable-ipv6-masquerade: "{{ cilium_enable_ipv6_masquerade }}"
{% endif %}
enable-bpf-masquerade: "{{ cilium_enable_bpf_masquerade }}"
@ -258,7 +221,6 @@ data:
enable-bpf-clock-probe: "{{ cilium_enable_bpf_clock_probe }}"
disable-cnp-status-updates: "{{ cilium_disable_cnp_status_updates }}"
{% if cilium_ip_masq_agent_enable %}
---
apiVersion: v1
@ -274,4 +236,4 @@ data:
{% endfor %}
masqLinkLocal: {{ cilium_masq_link_local|bool }}
resyncInterval: "{{ cilium_ip_masq_resync_interval }}"
{% endif %}
{% endif %}

60
roles/network_plugin/cilium/templates/cilium/cr.yml.j2
View File

@ -7,9 +7,6 @@ rules:
- apiGroups:
- networking.k8s.io
resources:
{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
- ingresses
{% endif %}
- networkpolicies
verbs:
- get
@ -28,34 +25,25 @@ rules:
resources:
- namespaces
- services
- nodes
- pods
- endpoints
{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
- componentstatuses
{% endif %}
- nodes
verbs:
- get
- list
- watch
{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
- apiGroups:
- extensions
resources:
- ingresses
verbs:
- create
- get
- list
- watch
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.7', '>') %}
{% if cilium_version | regex_replace('v') is version('1.12', '<') %}
- apiGroups:
- ""
resources:
- pods
- pods/finalizers
verbs:
- get
- list
- watch
- update
{% endif %}
- delete
- apiGroups:
- ""
resources:
@ -66,6 +54,7 @@ rules:
- list
- watch
- update
{% endif %}
- apiGroups:
- ""
resources:
@ -78,47 +67,45 @@ rules:
resources:
- customresourcedefinitions
verbs:
# Deprecated for removal in v1.10
- create
- get
- list
- watch
- update
# This is used when validating policies in preflight. This will need to stay
# until we figure out how to avoid "get" inside the preflight, and then
# should be removed ideally.
- get
- apiGroups:
- cilium.io
resources:
- ciliumnetworkpolicies
- ciliumnetworkpolicies/status
{% if cilium_version | regex_replace('v') is version('1.7', '>=') %}
- ciliumclusterwidenetworkpolicies
- ciliumclusterwidenetworkpolicies/status
{% endif %}
- ciliumendpoints
- ciliumendpoints/status
{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
- ciliumnodes
- ciliumnodes/status
- ciliumidentities
- ciliumidentities/status
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
- ciliumnetworkpolicies/finalizers
- ciliumclusterwidenetworkpolicies/finalizers
- ciliumendpoints/finalizers
- ciliumnodes/finalizers
- ciliumidentities/finalizers
- ciliumlocalredirectpolicies
- ciliumlocalredirectpolicies/status
- ciliumlocalredirectpolicies/finalizers
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
- ciliumegressnatpolicies
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
- ciliumendpointslices
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
- ciliumbgploadbalancerippools
- ciliumbgppeeringpolicies
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.11.5', '<') %}
- ciliumnetworkpolicies/finalizers
- ciliumclusterwidenetworkpolicies/finalizers
- ciliumendpoints/finalizers
- ciliumnodes/finalizers
- ciliumidentities/finalizers
- ciliumlocalredirectpolicies/finalizers
{% endif %}
verbs:
- '*'
@ -128,6 +115,7 @@ rules:
resources:
- ciliumclusterwideenvoyconfigs
- ciliumenvoyconfigs
- ciliumegressgatewaypolicies
verbs:
- list
- watch

5
roles/network_plugin/cilium/templates/cilium/crb.yml.j2
View File

@ -11,8 +11,3 @@ subjects:
- kind: ServiceAccount
name: cilium
namespace: kube-system
{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:nodes
{% endif %}

332
roles/network_plugin/cilium/templates/cilium/ds.yml.j2
View File

@ -2,37 +2,37 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
k8s-app: cilium
name: cilium
namespace: kube-system
labels:
k8s-app: cilium
spec:
selector:
matchLabels:
k8s-app: cilium
updateStrategy:
rollingUpdate:
# Specifies the maximum number of Pods that can be unavailable during the update process.
maxUnavailable: 2
type: RollingUpdate
template:
metadata:
annotations:
{% if cilium_enable_prometheus %}
prometheus.io/port: "9090"
prometheus.io/port: "{{ cilium_agent_scrape_port }}"
prometheus.io/scrape: "true"
{% endif %}
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]'
labels:
k8s-app: cilium
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: k8s-app
operator: In
values:
- cilium
topologyKey: kubernetes.io/hostname
containers:
- args:
- name: cilium-agent
image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
imagePullPolicy: {{ k8s_image_pull_policy }}
command:
- cilium-agent
args:
- --config-dir=/tmp/cilium/config-map
{% if cilium_mtu != "" %}
- --mtu={{ cilium_mtu }}
@ -44,54 +44,6 @@ spec:
- {{ flag }}
{% endfor %}
{% endif %}
command:
- cilium-agent
env:
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: CILIUM_K8S_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: CILIUM_CLUSTERMESH_CONFIG
value: /var/lib/cilium/clustermesh/
{% if cilium_kube_proxy_replacement == 'strict' %}
- name: KUBERNETES_SERVICE_HOST
value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
- name: KUBERNETES_SERVICE_PORT
value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
{% endif %}
{% for env_var in cilium_agent_extra_env_vars %}
- {{ env_var | to_nice_yaml(indent=2) | indent(10) }}
{% endfor %}
image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
limits:
cpu: {{ cilium_cpu_limit }}
memory: {{ cilium_memory_limit }}
requests:
cpu: {{ cilium_cpu_requests }}
memory: {{ cilium_memory_requests }}
lifecycle:
postStart:
exec:
command:
- "/cni-install.sh"
{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
- "--cni-exclusive={{ cilium_cni_exclusive }}"
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
- "--log-file={{ cilium_cni_log_file }}"
{% endif %}
preStop:
exec:
command:
- /cni-uninstall.sh
startupProbe:
httpGet:
host: '127.0.0.1'
@ -131,80 +83,109 @@ spec:
successThreshold: 1
failureThreshold: 3
timeoutSeconds: 5
name: cilium-agent
env:
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: CILIUM_K8S_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: CILIUM_CLUSTERMESH_CONFIG
value: /var/lib/cilium/clustermesh/
{% if cilium_kube_proxy_replacement == 'strict' %}
- name: KUBERNETES_SERVICE_HOST
value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
- name: KUBERNETES_SERVICE_PORT
value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
{% endif %}
{% for env_var in cilium_agent_extra_env_vars %}
- {{ env_var | to_nice_yaml(indent=2) | indent(10) }}
{% endfor %}
lifecycle:
postStart:
exec:
command:
- "/cni-install.sh"
- "--cni-exclusive={{ cilium_cni_exclusive | string | lower }}"
{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
- "--enable-debug={{ cilium_debug | string | lower }}"
- "--log-file={{ cilium_cni_log_file }}"
{% endif %}
preStop:
exec:
command:
- /cni-uninstall.sh
resources:
limits:
cpu: {{ cilium_cpu_limit }}
memory: {{ cilium_memory_limit }}
requests:
cpu: {{ cilium_cpu_requests }}
memory: {{ cilium_memory_requests }}
{% if cilium_enable_prometheus or cilium_enable_hubble_metrics %}
ports:
{% endif %}
{% if cilium_enable_prometheus %}
- containerPort: 9090
hostPort: 9090
name: prometheus
- name: prometheus
containerPort: {{ cilium_agent_scrape_port }}
hostPort: {{ cilium_agent_scrape_port }}
protocol: TCP
{% endif %}
{% if cilium_enable_hubble_metrics %}
- containerPort: 9091
hostPort: 9091
name: hubble-metrics
- name: hubble-metrics
containerPort: {{ cilium_hubble_scrape_port }}
hostPort: {{ cilium_hubble_scrape_port }}
protocol: TCP
{% endif %}
securityContext:
privileged: true
volumeMounts:
- mountPath: /sys/fs/bpf
name: bpf-maps
- name: bpf-maps
mountPath: /sys/fs/bpf
mountPropagation: Bidirectional
- mountPath: /var/run/cilium
name: cilium-run
- mountPath: /host/opt/cni/bin
name: cni-path
- mountPath: /host/etc/cni/net.d
name: etc-cni-netd
# pkg/workloads was depreca, removed in 1.7
# https://github.com/cilium/cilium/pull/9447
{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
{% if container_manager == 'docker' %}
- mountPath: /var/run/docker.sock
name: docker-socket
readOnly: true
{% else %}
- name: "{{ container_manager }}-socket"
mountPath: {{ cri_socket }}
readOnly: true
{% endif %}
{% endif %}
- name: cilium-run
mountPath: /var/run/cilium
- name: cni-path
mountPath: /host/opt/cni/bin
- name: etc-cni-netd
mountPath: /host/etc/cni/net.d
{% if cilium_identity_allocation_mode == "kvstore" %}
- mountPath: /var/lib/etcd-config
name: etcd-config-path
- name: etcd-config-path
mountPath: /var/lib/etcd-config
readOnly: true
- mountPath: "{{cilium_cert_dir}}"
name: etcd-secrets
- name: etcd-secrets
mountPath: "{{cilium_cert_dir}}"
readOnly: true
{% endif %}
- mountPath: /var/lib/cilium/clustermesh
name: clustermesh-secrets
- name: clustermesh-secrets
mountPath: /var/lib/cilium/clustermesh
readOnly: true
- mountPath: /tmp/cilium/config-map
name: cilium-config-path
- name: cilium-config-path
mountPath: /tmp/cilium/config-map
readOnly: true
{% if cilium_ip_masq_agent_enable %}
{% if cilium_ip_masq_agent_enable %}\
- name: ip-masq-agent
mountPath: /etc/config
readOnly: true
{% endif %}
# Needed to be able to load kernel modules
- mountPath: /lib/modules
name: lib-modules
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- mountPath: /run/xtables.lock
name: xtables-lock
- name: xtables-lock
mountPath: /run/xtables.lock
{% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
- mountPath: /etc/ipsec
name: cilium-ipsec-secrets
- name: cilium-ipsec-secrets
mountPath: /etc/ipsec
readOnly: true
{% endif %}
{% if cilium_hubble_install %}
- mountPath: /var/lib/cilium/tls/hubble
name: hubble-tls
- name: hubble-tls
mountPath: /var/lib/cilium/tls/hubble
readOnly: true
{% endif %}
{% for volume_mount in cilium_agent_extra_volume_mounts %}
@ -245,22 +226,50 @@ spec:
securityContext:
privileged: true
{% endif %}
- command:
{% if cilium_version | regex_replace('v') is version('1.11.7', '>=') %}
- name: apply-sysctl-overwrites
image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
imagePullPolicy: {{ k8s_image_pull_policy }}
env:
- name: BIN_PATH
value: /opt/cni/bin
command:
- sh
- -ec
# The statically linked Go program binary is invoked to avoid any
# dependency on utilities like sh that can be missing on certain
# distros installed on the underlying host. Copy the binary to the
# same directory where we install cilium cni plugin so that exec permissions
# are available.
- |
cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;
nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix";
rm /hostbin/cilium-sysctlfix
volumeMounts:
- name: hostproc
mountPath: /hostproc
- name: cni-path
mountPath: /hostbin
securityContext:
privileged: true
{% endif %}
- name: clean-cilium-state
image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
imagePullPolicy: {{ k8s_image_pull_policy }}
command:
- /init-container.sh
env:
- name: CILIUM_ALL_STATE
valueFrom:
configMapKeyRef:
key: clean-cilium-state
name: cilium-config
key: clean-cilium-state
optional: true
# CLEAN_CILIUM_BPF_STATE is deprecated in 1.6.
# https://github.com/cilium/cilium/pull/7478
- name: "{{ cilium_version | regex_replace('v') is version('1.6', '<')| ternary('CLEAN_CILIUM_BPF_STATE','CILIUM_BPF_STATE') }}"
- name: CILIUM_BPF_STATE
valueFrom:
configMapKeyRef:
key: clean-cilium-bpf-state
name: cilium-config
key: clean-cilium-bpf-state
optional: true
# Removed in 1.11 and up.
# https://github.com/cilium/cilium/commit/f7a3f59fd74983c600bfce9cac364b76d20849d9
@ -278,32 +287,25 @@ spec:
- name: KUBERNETES_SERVICE_PORT
value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}"
{% else %}
image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
{% endif %}
imagePullPolicy: {{ k8s_image_pull_policy }}
name: clean-cilium-state
securityContext:
privileged: true
volumeMounts:
- mountPath: /sys/fs/bpf
name: bpf-maps
- name: bpf-maps
mountPath: /sys/fs/bpf
{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
# Required to mount cgroup filesystem from the host to cilium agent pod
- name: cilium-cgroup
mountPath: {{ cilium_cgroup_host_root }}
mountPropagation: HostToContainer
{% endif %}
- mountPath: /var/run/cilium
name: cilium-run
- name: cilium-run
mountPath: /var/run/cilium
resources:
requests:
cpu: 100m
memory: 100Mi
priorityClassName: system-node-critical
restartPolicy: Always
priorityClassName: system-node-critical
serviceAccount: cilium
serviceAccountName: cilium
terminationGracePeriodSeconds: 1
@ -312,36 +314,26 @@ spec:
{% if cilium_identity_allocation_mode == "kvstore" %}
dnsPolicy: ClusterFirstWithHostNet
{% endif %}
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
k8s-app: cilium
tolerations:
- operator: Exists
volumes:
# To keep state between restarts / upgrades
- hostPath:
- name: cilium-run
hostPath:
path: /var/run/cilium
type: DirectoryOrCreate
name: cilium-run
# To keep state between restarts / upgrades for bpf maps
- hostPath:
- name: bpf-maps
hostPath:
path: /sys/fs/bpf
type: DirectoryOrCreate
name: bpf-maps
# pkg/workloads was deprecated in 1.6, removed in 1.7
# https://github.com/cilium/cilium/pull/9447
{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
{% if container_manager == 'docker' %}
# To read docker events from the node
- hostPath:
path: /var/run/docker.sock
type: Socket
name: docker-socket
{% else %}
# To read crio events from the node
- hostPath:
path: {{ cri_socket }}
type: Socket
name: {{ container_manager }}-socket
{% endif %}
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
# To mount cgroup2 filesystem on the host
- name: hostproc
@ -355,34 +347,34 @@ spec:
type: DirectoryOrCreate
{% endif %}
# To install cilium cni plugin in the host
- hostPath:
- name: cni-path
hostPath:
path: /opt/cni/bin
type: DirectoryOrCreate
name: cni-path
# To install cilium cni configuration in the host
- hostPath:
- name: etc-cni-netd
hostPath:
path: /etc/cni/net.d
type: DirectoryOrCreate
name: etc-cni-netd
# To be able to load kernel modules
- hostPath:
- name: lib-modules
hostPath:
path: /lib/modules
name: lib-modules
# To access iptables concurrently with other processes (e.g. kube-proxy)
- hostPath:
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
name: xtables-lock
{% if cilium_identity_allocation_mode == "kvstore" %}
# To read the etcd config stored in config maps
- configMap:
- name: etcd-config-path
configMap:
name: cilium-config
# note: the leading zero means this number is in octal representation: do not remove it
defaultMode: 0400
items:
- key: etcd-config
path: etcd.config
name: cilium-config
name: etcd-config-path
# To read the k8s etcd secrets in case the user might want to use TLS
- name: etcd-secrets
hostPath:
@ -391,21 +383,22 @@ spec:
# To read the clustermesh configuration
- name: clustermesh-secrets
secret:
secretName: cilium-clustermesh
# note: the leading zero means this number is in octal representation: do not remove it
defaultMode: 0400
optional: true
secretName: cilium-clustermesh
# To read the configuration from the config map
- configMap:
- name: cilium-config-path
configMap:
name: cilium-config
name: cilium-config-path
{% if cilium_ip_masq_agent_enable %}
- configMap:
- name: ip-masq-agent
configMap:
name: ip-masq-agent
optional: true
items:
- key: config
path: ip-masq-agent
name: ip-masq-agent
{% endif %}
{% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
- name: cilium-ipsec-secrets
@ -420,21 +413,12 @@ spec:
sources:
- secret:
name: hubble-server-certs
optional: true
items:
- key: ca.crt
path: client-ca.crt
- key: tls.crt
path: server.crt
- key: tls.key
path: server.key
optional: true
- configMap:
name: hubble-ca-cert
items:
- key: ca.crt
path: client-ca.crt
optional: true
{% endif %}
updateStrategy:
rollingUpdate:
# Specifies the maximum number of Pods that can be unavailable during the update process.
maxUnavailable: 2
type: RollingUpdate