update k8s binary & calico version

2022-09-08 21:10:33 +08:00 · 2022-09-08 21:10:33 +08:00 · 885c28dcac
parent 63ddc637d3
commit 885c28dcac
5 changed files with 105 additions and 45 deletions
--- a/example/config.yml
+++ b/example/config.yml
@ -121,7 +121,7 @@ CALICO_RR_ENABLED: false
 # CALICO_RR_NODES: ["192.168.1.1", "192.168.1.2"]
 CALICO_RR_NODES: []

-# [calico]更新支持calico 版本: [v3.3.x] [v3.4.x] [v3.8.x] [v3.15.x]
+# [calico]更新支持calico 版本: ["3.19", "3.23"]
 calico_ver: "__calico__"

 # [calico]calico 主版本
--- a/11
+++ b/11
@ -15,20 +15,20 @@ set -o errexit
 # default settings, can be overridden by cmd line options, see usage
 DOCKER_VER=20.10.16
 KUBEASZ_VER=3.3.1
-K8S_BIN_VER=v1.24.2
+K8S_BIN_VER=v1.25.0
 EXT_BIN_VER=1.2.0
 SYS_PKG_VER=0.4.3
 HARBOR_VER=v2.1.3
 REGISTRY_MIRROR=CN

 # images downloaded by default(with '-D')
-calicoVer=v3.19.4
+calicoVer=v3.23.3
 dnsNodeCacheVer=1.21.1
 corednsVer=1.9.3
 dashboardVer=v2.5.1
 dashboardMetricsScraperVer=v1.0.8
 metricsVer=v0.5.2
-pauseVer=3.7
+pauseVer=3.8

 # images not downloaded by default(only download  with '-X')
 ciliumVer=1.11.6
@ -300,17 +300,14 @@ function get_default_images() {
  # calico
  if [[ ! -f "$imageDir/calico_$calicoVer.tar" ]];then
    docker pull "calico/cni:$calicoVer" && \
-    docker pull "calico/pod2daemon-flexvol:$calicoVer" && \
    docker pull "calico/kube-controllers:$calicoVer" && \
    docker pull "calico/node:$calicoVer" && \
-    docker save -o "$imageDir/calico_$calicoVer.tar" "calico/cni:$calicoVer" "calico/kube-controllers:$calicoVer" "calico/node:$calicoVer" "calico/pod2daemon-flexvol:$calicoVer"
+    docker save -o "$imageDir/calico_$calicoVer.tar" "calico/cni:$calicoVer" "calico/kube-controllers:$calicoVer" "calico/node:$calicoVer"
  else
    docker load -i "$imageDir/calico_$calicoVer.tar"
  fi
  docker tag "calico/cni:$calicoVer" "easzlab.io.local:5000/calico/cni:$calicoVer"
  docker push "easzlab.io.local:5000/calico/cni:$calicoVer"
-  docker tag "calico/pod2daemon-flexvol:$calicoVer" "easzlab.io.local:5000/calico/pod2daemon-flexvol:$calicoVer"
-  docker push "easzlab.io.local:5000/calico/pod2daemon-flexvol:$calicoVer"
  docker tag "calico/kube-controllers:$calicoVer" "easzlab.io.local:5000/calico/kube-controllers:$calicoVer"
  docker push "easzlab.io.local:5000/calico/kube-controllers:$calicoVer"
  docker tag "calico/node:$calicoVer" "easzlab.io.local:5000/calico/node:$calicoVer"
--- a/roles/calico/templates/calico-v3.23.yaml.j2
+++ b/roles/calico/templates/calico-v3.23.yaml.j2
@ -1,7 +1,12 @@
 ---
+# download Release archive (https://github.com/projectcalico/calico/releases/download/v3.23.3/release-v3.23.3.tgz) 
+# Release notes: https://projectcalico.docs.tigera.io/archive/v3.23/release-notes/
+
 # Datastore: etcd, using Typha is redundant and not recommended.
 # Kubeasz uses cmd-line-way( kubectl create) to create etcd-secrets, see more in 'roles/calico/tasks/main.yml'

+# source from: release-v3.23.3/manifests/calico-etcd.yaml
+
 # Source: calico/templates/calico-config.yaml
 # This ConfigMap is used to configure a self-hosted Calico installation.
 kind: ConfigMap
@ -11,23 +16,21 @@ metadata:
  namespace: kube-system
 data:
  # Configure this with the location of your etcd cluster.
-  etcd_endpoints: "{{ ETCD_ENDPOINTS }}" 
+  etcd_endpoints: "{{ ETCD_ENDPOINTS }}"
  # If you're using TLS enabled etcd uncomment the following.
  # You must also populate the Secret below with these files.
  etcd_ca: "/calico-secrets/etcd-ca"
  etcd_cert: "/calico-secrets/etcd-cert"
-  etcd_key: "/calico-secrets/etcd-key"  
+  etcd_key: "/calico-secrets/etcd-key"

  # Typha is disabled.
  typha_service_name: "none"
  # Configure the backend to use.
  calico_backend: "{{ CALICO_NETWORKING_BACKEND }}"
  # Configure the MTU to use for workload interfaces and tunnels.
-  # - If Wireguard is enabled, set to your network MTU - 60
-  # - Otherwise, if VXLAN or BPF mode is enabled, set to your network MTU - 50
-  # - Otherwise, if IPIP is enabled, set to your network MTU - 20
-  # - Otherwise, if not using any encapsulation, set to your network MTU.
-  veth_mtu: "1440"
+  # By default, MTU is auto-detected, and explicitly setting this field should not be required.
+  # You can override auto-detection by providing a non-zero value.
+  veth_mtu: "0"

  # The CNI network configuration to install on each node. The special
  # values in this config will be automatically populated.
@ -39,6 +42,7 @@ data:
        {
          "type": "calico",
          "log_level": "info",
+          "log_file_path": "/var/log/calico/cni/cni.log",
          "etcd_endpoints": "{{ ETCD_ENDPOINTS }}",
          "etcd_key_file": "/etc/calico/ssl/calico-key.pem",
          "etcd_cert_file": "/etc/calico/ssl/calico.pem",
@ -120,6 +124,14 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: calico-node
 rules:
+  # Used for creating service account tokens to be used by the CNI plugin
+  - apiGroups: [""]
+    resources:
+      - serviceaccounts/token
+    resourceNames:
+      - calico-node
+    verbs:
+      - create
  # The CNI plugin needs to get pods, nodes, and namespaces.
  - apiGroups: [""]
    resources:
@ -128,6 +140,14 @@ rules:
      - namespaces
    verbs:
      - get
+  # EndpointSlices are used for Service-based network policy rule
+  # enforcement.
+  - apiGroups: ["discovery.k8s.io"]
+    resources:
+      - endpointslices
+    verbs:
+      - watch
+      - list
  - apiGroups: [""]
    resources:
      - endpoints
@ -209,8 +229,13 @@ spec:
        # This container installs the CNI binaries
        # and CNI network config file on each node.
        - name: install-cni
-          image: easzlab.io.local:5000/calico/cni:v3.15.3
-          command: ["/install-cni.sh"]
+          image: easzlab.io.local:5000/calico/cni:{{ calico_ver }}
+          command: ["/opt/cni/bin/install"]
+          envFrom:
+          - configMapRef:
+              # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+              name: kubernetes-services-endpoint
+              optional: true
          env:
            # Name of the CNI config file to create.
            - name: CNI_CONF_NAME
@ -245,21 +270,17 @@ spec:
              name: etcd-certs
          securityContext:
            privileged: true
-        # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
-        # to communicate with Felix over the Policy Sync API.
-        - name: flexvol-driver
-          image: easzlab.io.local:5000/calico/pod2daemon-flexvol:v3.15.3
-          volumeMounts:
-          - name: flexvol-driver-host
-            mountPath: /host/driver
-          securityContext:
-            privileged: true
      containers:
        # Runs calico-node container on each Kubernetes node. This
        # container programs network policy and routes on each
        # host.
        - name: calico-node
-          image: easzlab.io.local:5000/calico/node:v3.15.3
+          image: easzlab.io.local:5000/calico/node:{{ calico_ver }}
+          envFrom:
+          - configMapRef:
+              # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+              name: kubernetes-services-endpoint
+              optional: true
          env:
            # The location of the etcd cluster.
            - name: ETCD_ENDPOINTS
@ -313,6 +334,8 @@ spec:
 {% if CALICO_NETWORKING_BACKEND == "vxlan" %}
            - name: CALICO_IPV4POOL_VXLAN
              value: "Never"
+            - name: CALICO_IPV6POOL_VXLAN
+              value: "Never"
 {% endif %}
            # Set MTU for tunnel device used if ipip is enabled
            - name: FELIX_IPINIPMTU
@ -346,9 +369,6 @@ spec:
            # Disable IPv6 on Kubernetes.
            - name: FELIX_IPV6SUPPORT
              value: "false"
-            # Set Felix logging to "info"
-            - name: FELIX_LOGSEVERITYSCREEN
-              value: "warning"
            - name: FELIX_HEALTHENABLED
              value: "true"
            # Set Kubernetes NodePorts: If services do use NodePorts outside Calico’s expected range,
@ -362,6 +382,12 @@ spec:
          resources:
            requests:
              cpu: 250m
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - /bin/calico-node
+                - -shutdown
          livenessProbe:
            exec:
              command:
@ -373,6 +399,7 @@ spec:
            periodSeconds: 10
            initialDelaySeconds: 10
            failureThreshold: 6
+            timeoutSeconds: 10
          readinessProbe:
            exec:
              command:
@ -382,7 +409,12 @@ spec:
              - -bird-ready
 {% endif %}
            periodSeconds: 10
+            timeoutSeconds: 10
          volumeMounts:
+            # For maintaining CNI plugin API credentials.
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+              readOnly: false
            - mountPath: /lib/modules
              name: lib-modules
              readOnly: true
@ -399,6 +431,16 @@ spec:
              name: etcd-certs
            - name: policysync
              mountPath: /var/run/nodeagent
+            # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
+            # parent directory.
+            - name: sysfs
+              mountPath: /sys/fs/
+              # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host.
+              # If the host is known to mount that filesystem already then Bidirectional can be omitted.
+              mountPropagation: Bidirectional
+            - name: cni-log-dir
+              mountPath: /var/log/calico/cni
+              readOnly: true
      volumes:
        # Used by calico-node.
        - name: lib-modules
@ -414,6 +456,10 @@ spec:
          hostPath:
            path: /run/xtables.lock
            type: FileOrCreate
+        - name: sysfs
+          hostPath:
+            path: /sys/fs/
+            type: DirectoryOrCreate
        # Used to install CNI.
        - name: cni-bin-dir
          hostPath:
@ -421,6 +467,10 @@ spec:
        - name: cni-net-dir
          hostPath:
            path: /etc/cni/net.d
+        # Used to access CNI logs.
+        - name: cni-log-dir
+          hostPath:
+            path: /var/log/calico/cni
        # Mount in the etcd TLS secrets with mode 400.
        # See https://kubernetes.io/docs/concepts/configuration/secret/
        - name: etcd-certs
@ -432,11 +482,6 @@ spec:
          hostPath:
            type: DirectoryOrCreate
            path: /var/run/nodeagent
-        # Used to install Flex Volume Driver
-        - name: flexvol-driver-host
-          hostPath:
-            type: DirectoryOrCreate
-            path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
 ---

 apiVersion: v1
@ -485,7 +530,7 @@ spec:
      hostNetwork: true
      containers:
        - name: calico-kube-controllers
-          image: easzlab.io.local:5000/calico/kube-controllers:v3.15.3
+          image: easzlab.io.local:5000/calico/kube-controllers:{{ calico_ver }}
          env:
            # The location of the etcd cluster.
            - name: ETCD_ENDPOINTS
@ -518,18 +563,28 @@ spec:
            # Mount in the etcd TLS secrets.
            - mountPath: /calico-secrets
              name: etcd-certs
+          livenessProbe:
+            exec:
+              command:
+              - /usr/bin/check-status
+              - -l
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            failureThreshold: 6
+            timeoutSeconds: 10
          readinessProbe:
            exec:
              command:
              - /usr/bin/check-status
              - -r
+            periodSeconds: 10
      volumes:
        # Mount in the etcd TLS secrets with mode 400.
        # See https://kubernetes.io/docs/concepts/configuration/secret/
        - name: etcd-certs
          secret:
            secretName: calico-etcd-secrets
-            defaultMode: 0400
+            defaultMode: 0440

 ---

@ -539,3 +594,19 @@ metadata:
  name: calico-kube-controllers
  namespace: kube-system

+---
+
+# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: calico-kube-controllers
+  namespace: kube-system
+  labels:
+    k8s-app: calico-kube-controllers
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: calico-kube-controllers
--- a/roles/deploy/tasks/main.yml
+++ b/roles/deploy/tasks/main.yml
@ -55,10 +55,3 @@
 - name: ansible 控制端创建 kubectl 软链接
  file: src={{ base_dir }}/bin/kubectl dest=/usr/bin/kubectl state=link
  ignore_errors: true
-
- name: 写入环境变量$PATH
-  lineinfile:
-    dest: ~/.bashrc
-    state: present
-    regexp: 'custom PATH'
-    line: 'export PATH={{ base_dir }}/bin/:$PATH  # generated by kubeasz:custom PATH'
--- a/roles/prepare/tasks/ubuntu.yml
+++ b/roles/prepare/tasks/ubuntu.yml
@ -4,7 +4,6 @@
  with_items:
    - ufw
    - lxd
-    - lxd-client
    - lxcfs
    - lxc-common
  when: ansible_distribution == "Ubuntu"