[PR-Calico]Support calico 3.4.0 (#4102)

* Suport calico 3.4.0

Signed-off-by: wangxf1987 <xiaofeix.wang@gmail.com>

* Remove symlink + cni conflist template when 3.3.0+, handle Canal, addition of install-cni: sidecar(3.3.0) or initontainer(3.4.0), KUBECONFIG_FILEPATH, calico_cert_dir, advertise cluster ips

* scheduler.alpha.kubernetes.io/critical-pod deprecated since 1.12

inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml

@@ -18,3 +18,6 @@
 # not be specified in calico CNI config, so Calico will use built-in
 # defaults. The value should be a number, not a string.
 # calico_mtu: 1500
+
+# Advertise Cluster IPs
+# calico_advertise_cluster_ips: true

roles/download/defaults/main.yml

@@ -44,10 +44,10 @@ kube_image_repo: "gcr.io/google-containers"
 
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
-calico_version: "v3.1.3"
+calico_version: "v3.4.0"
-calico_ctl_version: "v3.1.3"
+calico_ctl_version: "v3.4.0"
-calico_cni_version: "v3.1.3"
+calico_cni_version: "v3.4.0"
-calico_policy_version: "v3.1.3"
+calico_policy_version: "v3.4.0"
 calico_rr_version: "v0.6.1"
 
 flannel_version: "v0.10.0"

roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: calico-kube-controllers
@@ -9,7 +9,7 @@ metadata:
 spec:
   replicas: 1
   strategy:
-    type: RollingUpdate
+    type: Recreate
   selector:
     matchLabels:
       kubernetes.io/cluster-service: "true"
@@ -22,11 +22,15 @@ spec:
         kubernetes.io/cluster-service: "true"
         k8s-app: calico-kube-controllers
     spec:
+      nodeSelector:
+        beta.kubernetes.io/os: linux
       hostNetwork: true
       serviceAccountName: calico-kube-controllers
       tolerations:
-        - effect: NoSchedule
+        - key: CriticalAddonsOnly
           operator: Exists
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-cluster-critical
 {% endif %}
@@ -50,6 +54,13 @@ spec:
               value: "{{ calico_cert_dir }}/cert.crt"
             - name: ETCD_KEY_FILE
               value: "{{ calico_cert_dir }}/key.pem"
+{% if calico_version is version('v3.3.0', '>=') %}
+          readinessProbe:
+            exec:
+              command:
+              - /usr/bin/check-status
+              - -r
+{% endif %}
           volumeMounts:
           - mountPath: {{ calico_cert_dir }}
             name: etcd-certs

roles/kubernetes/kubeadm/tasks/main.yml

@@ -99,7 +99,9 @@
     dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
     state: link
     force: yes
-  when: kube_network_plugin in ['calico','canal']
+  when:
+    - kube_network_plugin in ['calico','canal']
+    - calico_version is version('v3.3.0', '<')
 
 # FIXME(jjo): need to post-remove kube-proxy until https://github.com/kubernetes/kubeadm/issues/776
 # is fixed

roles/network_plugin/calico/defaults/main.yml

@@ -32,7 +32,7 @@ calico_node_cpu_requests: 150m
 calicoctl_memory_limit: 170M
 calicoctl_cpu_limit: 100m
 calicoctl_memory_requests: 32M
-calicoctl_cpu_requests: 50m
+calicoctl_cpu_requests: 250m
 
 # Enable Prometheus Metrics endpoint for felix
 calico_felix_prometheusmetricsenabled: "false"

roles/network_plugin/calico/tasks/install.yml

@@ -2,7 +2,7 @@
 - name: Calico | Write Calico cni config
   template:
     src: "cni-calico.conflist.j2"
-    dest: "/etc/cni/net.d/10-calico.conflist"
+    dest: "/etc/cni/net.d/{% if calico_version is version('v3.3.0', '>=') %}calico.conflist.template{% else %}10-calico.conflist{% endif %}"
     owner: kube
 
 - name: Calico | Create calico certs directory

roles/network_plugin/calico/templates/calico-cr.yml.j2

@@ -9,8 +9,21 @@ rules:
     resources:
       - pods
       - nodes
+      - namespaces
     verbs:
       - get
+  - apiGroups: [""]
+    resources:
+      - endpoints
+      - services
+    verbs:
+      - watch
+      - list
+  - apiGroups: [""]
+    resources:
+      - nodes/status
+    verbs:
+      - patch
   - apiGroups:
     - policy
     resourceNames:

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -31,14 +31,62 @@ spec:
       hostNetwork: true
       serviceAccountName: calico-node
       tolerations:
-        - operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists
         # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         - key: CriticalAddonsOnly
           operator: "Exists"
       # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
       # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
       terminationGracePeriodSeconds: 0
+{% if calico_version is version('v3.4.0', '>=') %}
+      initContainers:
+        # This container installs the Calico CNI binaries
+        # and CNI network config file on each node.
+        - name: install-cni
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+            # CNI binaries are already on the host
+            - name: UPDATE_CNI_BINARIES
+              value: "false"
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG_FILE
+              value: "/host/etc/cni/net.d/calico.conflist.template"
+            # Prevents the container from sleeping forever.
+            - name: SLEEP
+              value: "false"
+          volumeMounts:
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+{% endif %}
       containers:
+{% if calico_version is version('v3.3.0', '>=') and calico_version is version('v3.4.0', '<') %}
+        - name: install-cni
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+            # CNI binaries are already on the host
+            - name: UPDATE_CNI_BINARIES
+              value: "false"
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG_FILE
+              value: "/host/etc/cni/net.d/calico.conflist.template"
+            # Prevents the container from sleeping forever.
+            - name: SLEEP
+              value: "false"
+          volumeMounts:
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+{% endif %}
         # Runs calico/node container on each Kubernetes node.  This
         # container programs network policy and routes on each
         # host.
@@ -107,6 +155,10 @@ spec:
               value: "{{ calico_felix_prometheusgometricsenabled }}"
             - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED
               value: "{{ calico_felix_prometheusprocessmetricsenabled }}"
+{% if calico_version is version('v3.4.0', '>=') and calico_advertise_cluster_ips|default(false) %}
+            - name: CALICO_ADVERTISE_CLUSTER_IPS
+              value: "{{ kube_service_addresses }}"
+{% endif %}
             # Location of the CA certificate for etcd.
             - name: ETCD_CA_CERT_FILE
               valueFrom:
@@ -162,10 +214,18 @@ spec:
             initialDelaySeconds: 10
             failureThreshold: 6
           readinessProbe:
+{% if calico_version is version('v3.3.0', '<') %}
             httpGet:
               host: 127.0.0.1
               path: /readiness
               port: 9099
+{% else %}
+            exec:
+              command:
+              - /bin/calico-node
+              - -bird-ready
+              - -felix-ready
+{% endif %}
             periodSeconds: 10
           volumeMounts:
             - mountPath: /lib/modules
@@ -193,9 +253,6 @@ spec:
           hostPath:
             path: /var/lib/calico
         # Used to install CNI.
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
         - name: cni-net-dir
           hostPath:
             path: /etc/cni/net.d

roles/network_plugin/calico/templates/cni-calico.conflist.j2

@@ -10,9 +10,9 @@
     {% endif %}
       "type": "calico",
       "etcd_endpoints": "{{ etcd_access_addresses }}",
-      "etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem",
+      "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt",
-      "etcd_key_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem",
+      "etcd_key_file": "{{ calico_cert_dir }}/key.pem",
-      "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem",
+      "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt",
       "log_level": "info",
       "ipam": {
         "type": "calico-ipam",
@@ -28,7 +28,7 @@
       "mtu": {{ calico_mtu }},
 {%- endif %}
       "kubernetes": {
-        "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml"
+        "kubeconfig": "{% if calico_version is version('v3.3.0', '>=') %}__KUBECONFIG_FILEPATH__{% else %}{{ kube_config_dir }}/node-kubeconfig.yaml{% endif %}"
       }
     },
     {

roles/network_plugin/canal/tasks/main.yml

@@ -2,7 +2,7 @@
 - name: Canal | Write Canal cni config
   template:
     src: "cni-canal.conflist.j2"
-    dest: "/etc/cni/net.d/10-canal.conflist"
+    dest: "/etc/cni/net.d/{% if calico_version is version('v3.3.0', '>=') %}canal.conflist.template{% else %}10-canal.conflist{% endif %}"
     owner: kube
 
 - name: Canal | Create canal certs directory

roles/network_plugin/canal/templates/canal-node.yaml.j2

@@ -57,7 +57,49 @@ spec:
           hostPath:
             path: /run/xtables.lock
             type: FileOrCreate
+{% if calico_version is version('v3.4.0', '>=') %}
+      initContainers:
+        # This container installs the Calico CNI binaries
+        # and CNI network config file on each node.
+        - name: install-cni
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-canal.conflist"
+            # CNI binaries are already on the host
+            - name: UPDATE_CNI_BINARIES
+              value: "false"
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG_FILE
+              value: "/host/etc/cni/net.d/canal.conflist.template"
+            # Prevents the container from sleeping forever.
+            - name: SLEEP
+              value: "false"
+          volumeMounts:
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+{% endif %}
       containers:
+{% if calico_version is version('v3.3.0', '>=') and calico_version is version('v3.4.0', '<') %}
+        - name: install-cni
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-canal.conflist"
+            # CNI binaries are already on the host
+            - name: UPDATE_CNI_BINARIES
+              value: "false"
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG_FILE
+              value: "/host/etc/cni/net.d/canal.conflist.template"
+          volumeMounts:
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+{% endif %}
         # Runs the flannel daemon to enable vxlan networking between
         # container hosts.
         - name: flannel
@@ -209,9 +251,17 @@ spec:
             initialDelaySeconds: 10
             failureThreshold: 6
           readinessProbe:
+{% if calico_version is version('v3.3.0', '<')%}
             httpGet:
               path: /readiness
               port: 9099
+{% else %}
+            exec:
+              command:
+              - /bin/calico-node
+              - -bird-ready
+              - -felix-ready
+{% endif %}
             periodSeconds: 10
           volumeMounts:
             - mountPath: /lib/modules

roles/network_plugin/canal/templates/cni-canal.conflist.j2

@@ -15,7 +15,7 @@
           "type": "k8s"
         },
         "kubernetes": {
-          "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml"
+          "kubeconfig": "{% if calico_version is version('v3.3.0', '>=') %}__KUBECONFIG_FILEPATH__{% else %}{{ kube_config_dir }}/node-kubeconfig.yaml{% endif %}"
         }
       }
     },