Calico enable support for eBPF (#7618)

* Calico: align manifests with upstream * allow enabling typha prometheus metrics * Calico: enable eBPF support * manage the kubernetes-services-endpoint configmap * Calico: document the use of eBPF dataplane * Calico: improve checks before deployment * enforce disabling kube-proxy when using eBPF dataplane * ensure calico_version is supported
2021-06-07 14:58:39 +03:00 · 2021-06-07 14:58:39 +03:00 · ec0c0d4a28
parent 1739b27231
commit ec0c0d4a28
10 changed files with 231 additions and 12 deletions
--- a/docs/calico.md
+++ b/docs/calico.md
@ -260,3 +260,73 @@ calico_ipam_host_local: true
 ```

 Refer to Project Calico section [Using host-local IPAM](https://docs.projectcalico.org/reference/cni-plugin/configuration#using-host-local-ipam) for further information.
+
+## eBPF Support
+
+Calico supports eBPF for its data plane see [an introduction to the Calico eBPF Dataplane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/) for further information.
+
+Note that it is advisable to always use the latest version of Calico when using the eBPF dataplane.
+
+### Enabling eBPF support
+
+To enable the eBPF dataplane support ensure you add the following to your inventory. Note that the `kube-proxy` is incompatible with running Calico in eBPF mode and the kube-proxy should be removed from the system.
+
+```yaml
+calico_bpf_enabled: true
+kube_proxy_remove: true
+```
+
+### Cleaning up after kube-proxy
+
+Calico node cannot clean up after kube-proxy has run in ipvs mode. If you are converting an existing cluster to eBPF you will need to ensure the `kube-proxy` DaemonSet is deleted and that ipvs rules are cleaned.
+
+To check that kube-proxy was running in ipvs mode:
+
+```ShellSession
+# ipvsadm -l
+```
+
+To clean up any ipvs leftovers:
+
+```ShellSession
+# ipvsadm -C
+```
+
+### Calico access to the kube-api
+
+Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.
+
+Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](./ha-mode.md).
+
+If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints.
+
+```yaml
+loadbalancer_apiserver_localhost: true
+use_localhost_as_kubeapi_loadbalancer: true
+```
+
+### Tunneled versus Direct Server Return
+
+By default Calico usese Tunneled service mode but it can use direct server return (DSR) in order to optimize the return path for a service.
+
+To configure DSR:
+
+```yaml
+calico_bpf_service_mode: "DSR"
+```
+
+### eBPF Logging and Troubleshooting
+
+In order to enable Calico eBPF mode logging:
+
+```yaml
+calico_bpf_log_level: "Debug"
+```
+
+To view the logs you need to use the `tc` command to read the kernel trace buffer:
+
+```ShellSession
+tc exec bpf debug
+```
+
+Please see [Calico eBPF troubleshooting guide](https://docs.projectcalico.org/maintenance/troubleshoot/troubleshoot-ebpf#ebpf-program-debug-logs).
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@ -69,6 +69,7 @@ quay_image_repo: "quay.io"
 calico_version: "v3.18.4"
 calico_ctl_version: "{{ calico_version }}"
 calico_cni_version: "{{ calico_version }}"
+calico_flexvol_version: "{{ calico_version }}"
 calico_policy_version: "{{ calico_version }}"
 calico_typha_version: "{{ calico_version }}"
 typha_enabled: false
@ -439,6 +440,8 @@ calico_node_image_repo: "{{ quay_image_repo }}/calico/node"
 calico_node_image_tag: "{{ calico_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 calico_cni_image_repo: "{{ quay_image_repo }}/calico/cni"
 calico_cni_image_tag: "{{ calico_cni_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
+calico_flexvol_image_repo: "{{ quay_image_repo }}/calico/pod2daemon-flexvol"
+calico_flexvol_image_tag: "{{ calico_flexvol_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 calico_policy_image_repo: "{{ quay_image_repo }}/calico/kube-controllers"
 calico_policy_image_tag: "{{ calico_policy_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 calico_typha_image_repo: "{{ quay_image_repo }}/calico/typha"
--- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
+++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
@ -47,6 +47,25 @@
    - kube_network_plugin == 'calico'
    - not ignore_assert_errors

+- name: Stop if supported Calico versions
+  assert:
+    that:
+      - "calico_version in calico_crds_archive_checksums.keys()"
+    msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.keys() }}"
+  when:
+    - kube_network_plugin == 'calico'
+    - not ignore_assert_errors
+
+- name: Stop if kube-proxy is enabled when using eBPF dataplane
+  assert:
+    that:
+      - kube_proxy_remove
+    msg: "kube-proxy needs to be disabled when using Calico with eBPF dataplane"
+  when:
+    - calico_bpf_enabled | default(false)
+    - kube_network_plugin == 'calico'
+    - not ignore_assert_errors
+
 - name: Stop if unsupported version of Kubernetes
  assert:
    that: kube_version is version(kube_version_min_required, '>=')
--- a/roles/network_plugin/calico/defaults/main.yml
+++ b/roles/network_plugin/calico/defaults/main.yml
@ -32,6 +32,12 @@ calico_advertise_service_external_ips: []
 # Adveritse Service LoadBalancer IPs
 calico_advertise_service_loadbalancer_ips: []

+# Calico eBPF support
+calico_bpf_enabled: false
+calico_bpf_log_level: ""
+# Valid option for service mode: Tunnel (default), DSR=Direct Server Return
+calico_bpf_service_mode: Tunnel
+
 # Limits for apps
 calico_node_memory_limit: 500M
 calico_node_cpu_limit: 300m
@ -91,6 +97,8 @@ kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem

 # Use typha (only with kdd)
 typha_enabled: false
+typha_prometheusmetricsenabled: false
+typha_prometheusmetricsport: 9093

 # Scaling typha: 1 replica per 100 nodes is adequate
 # Number of typha replicas
@ -106,3 +114,7 @@ calico_feature_control: {}

 # Calico default BGP port
 calico_bgp_listen_port: 179
+
+# Calico FelixConfiguration options
+calico_felix_reporting_interval: 0s
+calico_felix_log_severity_screen: Info
--- a/roles/network_plugin/calico/tasks/install.yml
+++ b/roles/network_plugin/calico/tasks/install.yml
@ -131,6 +131,27 @@
    - inventory_hostname in groups['kube_control_plane']
    - calico_datastore == "kdd"

+- name: Calico | Configure calico FelixConfiguration
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      { "kind": "FelixConfiguration",
+        "apiVersion": "projectcalico.org/v3",
+        "metadata": {
+          "name": "default",
+        },
+        "spec": {
+          "ipipEnabled": {{ calico_ipip_mode != 'Never' | bool }},
+          "reportingInterval": "{{ calico_felix_reporting_interval }}",
+          "bpfLogLevel": "{{ calico_bpf_log_level }}",
+          "bpfEnabled": {{ calico_bpf_enabled | bool }},
+          "bpfExternalServiceMode": "{{ calico_bpf_service_mode }}",
+          "logSeverityScreen": "{{ calico_felix_log_severity_screen }}" }}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
 - name: Calico | Configure calico network pool
  command:
    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
@ -302,6 +323,7 @@
    - {name: calico, file: calico-node-sa.yml, type: sa}
    - {name: calico, file: calico-cr.yml, type: clusterrole}
    - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
+    - {name: kubernetes-services-endpoint, file: kubernetes-services-endpoint.yml, type: cm }
  register: calico_node_manifests
  when:
    - inventory_hostname in groups['kube_control_plane']
--- a/roles/network_plugin/calico/templates/calico-cr.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-cr.yml.j2
@ -28,6 +28,7 @@ rules:
    resources:
      - nodes/status
    verbs:
+      # Needed for clearing NodeNetworkUnavailable flag.
      - patch
 {% if calico_datastore == "etcd" %}
  - apiGroups:
--- a/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-node.yml.j2
@ -44,6 +44,11 @@ spec:
        - name: upgrade-ipam
          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
          command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
+          envFrom:
+            - configMapRef:
+                # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+                name: kubernetes-services-endpoint
+                optional: true
          env:
            - name: KUBERNETES_NODE_NAME
              valueFrom:
@ -94,12 +99,26 @@ spec:
              name: cni-bin-dir
          securityContext:
            privileged: true
+        # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
+        # to communicate with Felix over the Policy Sync API.
+        - name: flexvol-driver
+          image: {{ calico_flexvol_image_repo }}:{{ calico_flexvol_image_tag }}
+          volumeMounts:
+            - name: flexvol-driver-host
+              mountPath: /host/driver
+          securityContext:
+            privileged: true
      containers:
        # Runs calico/node container on each Kubernetes node.  This
        # container programs network policy and routes on each
        # host.
        - name: calico-node
          image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
+          envFrom:
+            - configMapRef:
+                # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+                name: kubernetes-services-endpoint
+                optional: true
          env:
            # The location of the Calico etcd cluster.
 {% if calico_datastore == "etcd" %}
@ -231,8 +250,6 @@ spec:
 {% if calico_ip_auto_method is defined %}
            - name: IP_AUTODETECTION_METHOD
              value: "{{ calico_ip_auto_method }}"
-            - name: IP
-              value: "autodetect"
 {% else %}
            - name: NODEIP
              valueFrom:
@ -240,9 +257,9 @@ spec:
                  fieldPath: status.hostIP
            - name: IP_AUTODETECTION_METHOD
              value: "can-reach=$(NODEIP)"
+{% endif %}
            - name: IP
              value: "autodetect"
-{% endif %}
 {% if enable_dual_stack_networks %}
            - name: IP6
              value: autodetect
@ -286,10 +303,10 @@ spec:
 {% if calico_network_backend|default("bird") == "bird" %}
              - -bird-live
 {% endif %}
-            initialDelaySeconds: 5
+            periodSeconds: 10
+            initialDelaySeconds: 10
            failureThreshold: 6
          readinessProbe:
-            failureThreshold: 6
            exec:
              command:
              - /bin/calico-node
@ -297,18 +314,22 @@ spec:
              - -bird-ready
 {% endif %}
              - -felix-ready
+            periodSeconds: 10
+            failureThreshold: 6
          volumeMounts:
            - mountPath: /lib/modules
              name: lib-modules
              readOnly: true
            - mountPath: /var/run/calico
              name: var-run-calico
+              readOnly: false
            - mountPath: /var/lib/calico
              name: var-lib-calico
              readOnly: false
 {% if calico_datastore == "etcd" %}
            - mountPath: /calico-secrets
              name: etcd-certs
+              readOnly: true
 {% endif %}
            - name: xtables-lock
              mountPath: /run/xtables.lock
@ -324,7 +345,20 @@ spec:
              mountPath: /etc/typha-ca/ca.crt
              readOnly: true
 {% endif %}
-
+            - name: policysync
+              mountPath: /var/run/nodeagent
+{% if calico_bpf_enabled %}
+            # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
+            # parent directory.
+            - name: sysfs
+              mountPath: /sys/fs/
+              # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host.
+              # If the host is known to mount that filesystem already then Bidirectional can be omitted.
+              mountPropagation: Bidirectional
+{% endif %}
+            - name: cni-log-dir
+              mountPath: /var/log/calico/cni
+              readOnly: true
      volumes:
        # Used by calico/node.
        - name: lib-modules
@ -375,6 +409,26 @@ spec:
          hostPath:
            path: "/etc/kubernetes/ssl/"
 {% endif %}
+{% if calico_bpf_enabled %}
+        - name: sysfs
+          hostPath:
+            path: /sys/fs/
+            type: DirectoryOrCreate
+{% endif %}
+        # Used to access CNI logs.
+        - name: cni-log-dir
+          hostPath:
+            path: /var/log/calico/cni
+        # Used to create per-pod Unix Domain Sockets
+        - name: policysync
+          hostPath:
+            type: DirectoryOrCreate
+            path: /var/run/nodeagent
+        # Used to install Flex Volume Driver
+        - name: flexvol-driver-host
+          hostPath:
+            type: DirectoryOrCreate
+            path: "{{ kubelet_flexvolumes_plugins_dir | default('/usr/libexec/kubernetes/kubelet-plugins/volume/exec') }}/nodeagent~uds"
  updateStrategy:
    rollingUpdate:
      maxUnavailable: {{ serial | default('20%') }}
--- a/roles/network_plugin/calico/templates/calico-typha.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-typha.yml.j2
@ -46,6 +46,10 @@ spec:
        k8s-app: calico-typha
      annotations:
        cluster-autoscaler.kubernetes.io/safe-to-evict: 'true'
+{% if typha_prometheusmetricsenabled %}
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: "{{ typha_prometheusmetricsport }}"
+{% endif %}
    spec:
      nodeSelector:
        kubernetes.io/os: linux
@ -61,6 +65,9 @@ spec:
      # as a host-networked pod.
      serviceAccountName: calico-node
      priorityClassName: system-cluster-critical
+      # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573
+      securityContext:
+        fsGroup: 65534
      containers:
      - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }}
        name: calico-typha
@ -68,6 +75,11 @@ spec:
        - containerPort: 5473
          name: calico-typha
          protocol: TCP
+        envFrom:
+        - configMapRef:
+            # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+            name: kubernetes-services-endpoint
+            optional: true
        env:
          # Enable "info" logging by default.  Can be set to "debug" to increase verbosity.
          - name: TYPHA_LOGSEVERITYSCREEN
@ -105,13 +117,14 @@ spec:
            name: cacert
            readOnly: true
 {% endif %}
-          # Uncomment these lines to enable prometheus metrics.  Since Typha is host-networked,
+{% if typha_prometheusmetricsenabled %}
+          # Since Typha is host-networked,
          # this opens a port on the host, which may need to be secured.
-          #- name: TYPHA_PROMETHEUSMETRICSENABLED
-          #  value: "true"
-          #- name: TYPHA_PROMETHEUSMETRICSPORT
-          #  value: "9093"
-
+          - name: TYPHA_PROMETHEUSMETRICSENABLED
+            value: "true"
+          - name: TYPHA_PROMETHEUSMETRICSPORT
+            value: "{{ typha_prometheusmetricsport }}"
+{% endif %}
          # Needed for version >=3.7 when the 'host-local' ipam is used
          # Should never happen given templates/cni-calico.conflist.j2
          # Configure route aggregation based on pod CIDR.
--- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2
+++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2
@ -64,6 +64,12 @@
      "capabilities": {
        "portMappings": true
      }
+    },
+    {
+      "type":"bandwidth",
+      "capabilities": {
+        "bandwidth": true
+      }
    }
  ]
 }
--- a/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2
+++ b/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2
@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: kube-system
+  name: kubernetes-services-endpoint
+data:
+{% if calico_bpf_enabled %}
+{% if loadbalancer_apiserver is defined %}
+  KUBERNETES_SERVICE_HOST: "{{ apiserver_loadbalancer_domain_name }}"
+  KUBERNETES_SERVICE_PORT: "{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
+{%- elif use_localhost_as_kubeapi_loadbalancer|default(False)|bool %}
+  KUBERNETES_SERVICE_HOST: "127.0.0.1"
+  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}"
+{%- else %}
+  KUBERNETES_SERVICE_HOST: "{{ first_kube_master }}"
+  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}"
+{%- endif %}
+{% endif %}