[PR-Calico]Support calico 3.4.0 (#4102)

* Suport calico 3.4.0 Signed-off-by: wangxf1987 <xiaofeix.wang@gmail.com> * Remove symlink + cni conflist template when 3.3.0+, handle Canal, addition of install-cni: sidecar(3.3.0) or initontainer(3.4.0), KUBECONFIG_FILEPATH, calico_cert_dir, advertise cluster ips * scheduler.alpha.kubernetes.io/critical-pod deprecated since 1.12
2019-01-29 03:03:49 +08:00 · 2019-01-29 03:03:49 +08:00 · a096761306
parent d790ec96d8
commit a096761306
12 changed files with 156 additions and 20 deletions
--- a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml
+++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml
@ -18,3 +18,6 @@
 # not be specified in calico CNI config, so Calico will use built-in
 # defaults. The value should be a number, not a string.
 # calico_mtu: 1500
+
+# Advertise Cluster IPs
+# calico_advertise_cluster_ips: true
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@ -44,10 +44,10 @@ kube_image_repo: "gcr.io/google-containers"

 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
-calico_version: "v3.1.3"
-calico_ctl_version: "v3.1.3"
-calico_cni_version: "v3.1.3"
-calico_policy_version: "v3.1.3"
+calico_version: "v3.4.0"
+calico_ctl_version: "v3.4.0"
+calico_cni_version: "v3.4.0"
+calico_policy_version: "v3.4.0"
 calico_rr_version: "v0.6.1"

 flannel_version: "v0.10.0"
--- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: calico-kube-controllers
@ -9,7 +9,7 @@ metadata:
 spec:
  replicas: 1
  strategy:
-    type: RollingUpdate
+    type: Recreate
  selector:
    matchLabels:
      kubernetes.io/cluster-service: "true"
@ -22,11 +22,15 @@ spec:
        kubernetes.io/cluster-service: "true"
        k8s-app: calico-kube-controllers
    spec:
+      nodeSelector:
+        beta.kubernetes.io/os: linux
      hostNetwork: true
      serviceAccountName: calico-kube-controllers
      tolerations:
-        - effect: NoSchedule
+        - key: CriticalAddonsOnly
          operator: Exists
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
 {% if kube_version is version('v1.11.1', '>=') %}
      priorityClassName: system-cluster-critical
 {% endif %}
@ -50,6 +54,13 @@ spec:
              value: "{{ calico_cert_dir }}/cert.crt"
            - name: ETCD_KEY_FILE
              value: "{{ calico_cert_dir }}/key.pem"
+{% if calico_version is version('v3.3.0', '>=') %}
+          readinessProbe:
+            exec:
+              command:
+              - /usr/bin/check-status
+              - -r
+{% endif %}
          volumeMounts:
          - mountPath: {{ calico_cert_dir }}
            name: etcd-certs
--- a/roles/kubernetes/kubeadm/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/tasks/main.yml
@ -99,7 +99,9 @@
    dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
    state: link
    force: yes
-  when: kube_network_plugin in ['calico','canal']
+  when:
+    - kube_network_plugin in ['calico','canal']
+    - calico_version is version('v3.3.0', '<')

 # FIXME(jjo): need to post-remove kube-proxy until https://github.com/kubernetes/kubeadm/issues/776
 # is fixed
--- a/roles/network_plugin/calico/defaults/main.yml
+++ b/roles/network_plugin/calico/defaults/main.yml
@ -32,7 +32,7 @@ calico_node_cpu_requests: 150m
 calicoctl_memory_limit: 170M
 calicoctl_cpu_limit: 100m
 calicoctl_memory_requests: 32M
-calicoctl_cpu_requests: 50m
+calicoctl_cpu_requests: 250m

 # Enable Prometheus Metrics endpoint for felix
 calico_felix_prometheusmetricsenabled: "false"
--- a/roles/network_plugin/calico/tasks/install.yml
+++ b/roles/network_plugin/calico/tasks/install.yml
@ -2,7 +2,7 @@
 - name: Calico | Write Calico cni config
  template:
    src: "cni-calico.conflist.j2"
-    dest: "/etc/cni/net.d/10-calico.conflist"
+    dest: "/etc/cni/net.d/{% if calico_version is version('v3.3.0', '>=') %}calico.conflist.template{% else %}10-calico.conflist{% endif %}"
    owner: kube

 - name: Calico | Create calico certs directory
--- a/roles/network_plugin/calico/templates/calico-cr.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-cr.yml.j2
@ -9,8 +9,21 @@ rules:
    resources:
      - pods
      - nodes
+      - namespaces
    verbs:
      - get
+  - apiGroups: [""]
+    resources:
+      - endpoints
+      - services
+    verbs:
+      - watch
+      - list
+  - apiGroups: [""]
+    resources:
+      - nodes/status
+    verbs:
+      - patch
  - apiGroups:
    - policy
    resourceNames:
--- a/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-node.yml.j2
@ -31,14 +31,62 @@ spec:
      hostNetwork: true
      serviceAccountName: calico-node
      tolerations:
-        - operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists
        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
        - key: CriticalAddonsOnly
          operator: "Exists"
      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
      terminationGracePeriodSeconds: 0
+{% if calico_version is version('v3.4.0', '>=') %}
+      initContainers:
+        # This container installs the Calico CNI binaries
+        # and CNI network config file on each node.
+        - name: install-cni
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+            # CNI binaries are already on the host
+            - name: UPDATE_CNI_BINARIES
+              value: "false"
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG_FILE
+              value: "/host/etc/cni/net.d/calico.conflist.template"
+            # Prevents the container from sleeping forever.
+            - name: SLEEP
+              value: "false"
+          volumeMounts:
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+{% endif %}
      containers:
+{% if calico_version is version('v3.3.0', '>=') and calico_version is version('v3.4.0', '<') %}
+        - name: install-cni
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+            # CNI binaries are already on the host
+            - name: UPDATE_CNI_BINARIES
+              value: "false"
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG_FILE
+              value: "/host/etc/cni/net.d/calico.conflist.template"
+            # Prevents the container from sleeping forever.
+            - name: SLEEP
+              value: "false"
+          volumeMounts:
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+{% endif %}
        # Runs calico/node container on each Kubernetes node.  This
        # container programs network policy and routes on each
        # host.
@ -107,6 +155,10 @@ spec:
              value: "{{ calico_felix_prometheusgometricsenabled }}"
            - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED
              value: "{{ calico_felix_prometheusprocessmetricsenabled }}"
+{% if calico_version is version('v3.4.0', '>=') and calico_advertise_cluster_ips|default(false) %}
+            - name: CALICO_ADVERTISE_CLUSTER_IPS
+              value: "{{ kube_service_addresses }}"
+{% endif %}
            # Location of the CA certificate for etcd.
            - name: ETCD_CA_CERT_FILE
              valueFrom:
@ -162,10 +214,18 @@ spec:
            initialDelaySeconds: 10
            failureThreshold: 6
          readinessProbe:
+{% if calico_version is version('v3.3.0', '<') %}
            httpGet:
              host: 127.0.0.1
              path: /readiness
              port: 9099
+{% else %}
+            exec:
+              command:
+              - /bin/calico-node
+              - -bird-ready
+              - -felix-ready
+{% endif %}
            periodSeconds: 10
          volumeMounts:
            - mountPath: /lib/modules
@ -193,9 +253,6 @@ spec:
          hostPath:
            path: /var/lib/calico
        # Used to install CNI.
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
        - name: cni-net-dir
          hostPath:
            path: /etc/cni/net.d
--- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2
+++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2
@ -10,9 +10,9 @@
    {% endif %}
      "type": "calico",
      "etcd_endpoints": "{{ etcd_access_addresses }}",
-      "etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem",
-      "etcd_key_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem",
-      "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem",
+      "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt",
+      "etcd_key_file": "{{ calico_cert_dir }}/key.pem",
+      "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt",
      "log_level": "info",
      "ipam": {
        "type": "calico-ipam",
@ -28,7 +28,7 @@
      "mtu": {{ calico_mtu }},
 {%- endif %}
      "kubernetes": {
-        "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml"
+        "kubeconfig": "{% if calico_version is version('v3.3.0', '>=') %}__KUBECONFIG_FILEPATH__{% else %}{{ kube_config_dir }}/node-kubeconfig.yaml{% endif %}"
      }
    },
    {
--- a/roles/network_plugin/canal/tasks/main.yml
+++ b/roles/network_plugin/canal/tasks/main.yml
@ -2,7 +2,7 @@
 - name: Canal | Write Canal cni config
  template:
    src: "cni-canal.conflist.j2"
-    dest: "/etc/cni/net.d/10-canal.conflist"
+    dest: "/etc/cni/net.d/{% if calico_version is version('v3.3.0', '>=') %}canal.conflist.template{% else %}10-canal.conflist{% endif %}"
    owner: kube

 - name: Canal | Create canal certs directory
--- a/roles/network_plugin/canal/templates/canal-node.yaml.j2
+++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2
@ -57,7 +57,49 @@ spec:
          hostPath:
            path: /run/xtables.lock
            type: FileOrCreate
+{% if calico_version is version('v3.4.0', '>=') %}
+      initContainers:
+        # This container installs the Calico CNI binaries
+        # and CNI network config file on each node.
+        - name: install-cni
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-canal.conflist"
+            # CNI binaries are already on the host
+            - name: UPDATE_CNI_BINARIES
+              value: "false"
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG_FILE
+              value: "/host/etc/cni/net.d/canal.conflist.template"
+            # Prevents the container from sleeping forever.
+            - name: SLEEP
+              value: "false"
+          volumeMounts:
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+{% endif %}
      containers:
+{% if calico_version is version('v3.3.0', '>=') and calico_version is version('v3.4.0', '<') %}
+        - name: install-cni
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-canal.conflist"
+            # CNI binaries are already on the host
+            - name: UPDATE_CNI_BINARIES
+              value: "false"
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG_FILE
+              value: "/host/etc/cni/net.d/canal.conflist.template"
+          volumeMounts:
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+{% endif %}
        # Runs the flannel daemon to enable vxlan networking between
        # container hosts.
        - name: flannel
@ -209,9 +251,17 @@ spec:
            initialDelaySeconds: 10
            failureThreshold: 6
          readinessProbe:
+{% if calico_version is version('v3.3.0', '<')%}
            httpGet:
              path: /readiness
              port: 9099
+{% else %}
+            exec:
+              command:
+              - /bin/calico-node
+              - -bird-ready
+              - -felix-ready
+{% endif %}
            periodSeconds: 10
          volumeMounts:
            - mountPath: /lib/modules
--- a/roles/network_plugin/canal/templates/cni-canal.conflist.j2
+++ b/roles/network_plugin/canal/templates/cni-canal.conflist.j2
@ -15,7 +15,7 @@
          "type": "k8s"
        },
        "kubernetes": {
-          "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml"
+          "kubeconfig": "{% if calico_version is version('v3.3.0', '>=') %}__KUBECONFIG_FILEPATH__{% else %}{{ kube_config_dir }}/node-kubeconfig.yaml{% endif %}"
        }
      }
    },