All CNIs: support ANY toleration. (#3391)

Before, Nodes tainted with NoExecute policy did not have calico/weave Pod.
Network pod should run on all nodes whatever happens on a specific node.

Also always set the Pods to be critical.
Also remove deprecated scheduler.alpha.kubernetes.io/tolerations annotations.

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -18,6 +18,7 @@ spec:
       labels:
         k8s-app: calico-node
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
         kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
     spec:
@@ -27,8 +28,10 @@ spec:
       hostNetwork: true
       serviceAccountName: calico-node
       tolerations:
-        - effect: NoSchedule
+        - operator: Exists
-          operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
       # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
       terminationGracePeriodSeconds: 0

roles/network_plugin/canal/templates/canal-node.yaml.j2

@@ -13,8 +13,8 @@ spec:
   template:
     metadata:
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
-        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
       labels:
         k8s-app: canal-node
     spec:
@@ -24,8 +24,10 @@ spec:
       hostNetwork: true
       serviceAccountName: canal
       tolerations:
-        - effect: NoSchedule
+        - operator: Exists
-          operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       volumes:
         # Used by calico/node.
         - name: lib-modules

roles/network_plugin/cilium/templates/cilium-ds.yml.j2

@@ -27,8 +27,6 @@ spec:
         # gets priority scheduling.
         # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
         scheduler.alpha.kubernetes.io/critical-pod: ''
-        scheduler.alpha.kubernetes.io/tolerations: >-
-          [{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]
 {% if cilium_enable_prometheus %}
         prometheus.io/scrape: "true"
         prometheus.io/port: "9090"
@@ -225,11 +223,7 @@ spec:
 
       restartPolicy: Always
       tolerations:
-        - effect: NoSchedule
+        - operator: Exists
-          key: node-role.kubernetes.io/master
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        - effect: NoSchedule
-          key: node.cloudprovider.kubernetes.io/uninitialized
-          value: "true"
-        # Mark cilium's pod as critical for rescheduling
         - key: CriticalAddonsOnly
           operator: "Exists"

roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2

@@ -16,6 +16,7 @@ spec:
       labels:
         k8s-app: contiv-api-proxy
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -28,8 +29,10 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: "true"
       tolerations:
-      - key: node-role.kubernetes.io/master
+        - operator: Exists
-        effect: NoSchedule
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       serviceAccountName: contiv-netmaster
       containers:
         - name: contiv-api-proxy

roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2

@@ -14,6 +14,9 @@ spec:
     metadata:
       labels:
         k8s-app: contiv-cleanup
+      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
       priorityClassName: system-node-critical
@@ -21,8 +24,10 @@ spec:
       hostNetwork: true
       hostPID: true
       tolerations:
-      - key: node-role.kubernetes.io/master
+        - operator: Exists
-        effect: NoSchedule
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       serviceAccountName: contiv-netplugin
       containers:
       - name: contiv-ovs-cleanup

roles/network_plugin/contiv/templates/contiv-etcd.yml.j2

@@ -25,8 +25,10 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: "true"
       tolerations:
-        - key: node-role.kubernetes.io/master
+        - operator: Exists
-          effect: NoSchedule
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       initContainers:
         - name: contiv-etcd-init
           image: {{ contiv_etcd_init_image_repo }}:{{ contiv_etcd_init_image_tag }}

roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2

@@ -16,6 +16,7 @@ spec:
       labels:
         k8s-app: contiv-netmaster
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -28,8 +29,10 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: "true"
       tolerations:
-      - key: node-role.kubernetes.io/master
+        - operator: Exists
-        effect: NoSchedule
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       serviceAccountName: contiv-netmaster
       containers:
         - name: contiv-netmaster

roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2

@@ -20,6 +20,7 @@ spec:
       labels:
         k8s-app: contiv-netplugin
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -28,8 +29,10 @@ spec:
       hostNetwork: true
       hostPID: true
       tolerations:
-      - key: node-role.kubernetes.io/master
+        - operator: Exists
-        effect: NoSchedule
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       serviceAccountName: contiv-netplugin
       initContainers:
         - name: contiv-netplugin-init

roles/network_plugin/contiv/templates/contiv-ovs.yml.j2

@@ -17,6 +17,7 @@ spec:
       labels:
         k8s-app: contiv-ovs
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -25,8 +26,10 @@ spec:
       hostNetwork: true
       hostPID: true
       tolerations:
-      - key: node-role.kubernetes.io/master
+        - operator: Exists
-        effect: NoSchedule
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       containers:
       # Runs ovs containers on each Kubernetes node.
       - name: contiv-ovsdb-server

roles/network_plugin/flannel/templates/cni-flannel.yml.j2

@@ -51,6 +51,9 @@ spec:
       labels:
         tier: node
         k8s-app: flannel
+      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
       priorityClassName: system-node-critical
@@ -108,9 +111,10 @@ spec:
           mountPath: /host/opt/cni/bin/
       hostNetwork: true
       tolerations:
-      - key: node-role.kubernetes.io/master
+        - operator: Exists
-        operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        effect: NoSchedule
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       volumes:
         - name: run
           hostPath:

roles/network_plugin/weave/templates/weave-net.yml.j2

@@ -114,6 +114,9 @@ items:
         metadata:
           labels:
             name: weave-net
+          annotations:
+            # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+            scheduler.alpha.kubernetes.io/critical-pod: ''
         spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
           priorityClassName: system-node-critical
@@ -224,8 +227,10 @@ items:
             seLinuxOptions: {}
           serviceAccountName: weave-net
           tolerations:
-            - effect: NoSchedule
+            - operator: Exists
-              operator: Exists
+            # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+            - key: CriticalAddonsOnly
+              operator: "Exists"
           volumes:
             - name: weavedb
               hostPath: