From e3dcd96301ee01ded95da9ae3a3091c99a3238ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20de=20Saint=20Martin?=
 <cedric@desaintmartin.fr>
Date: Thu, 9 Aug 2018 17:42:53 +0200
Subject: [PATCH] kubedns & kubedns-autoscaler: Stick to master nodes. (#2909)

* kubedns & kubedns-autoscaler: Stick to master nodes.

 - Tolerate only master nodes and not any NoSchedule taint
 - Pods are on different nodes
 - Pods are required to be on a master node.

* kubedns: use soft nodeAffinity.

Prefer to be on a master node, don't require.

* coredns: Stick to (different) master nodes.

     - Pods are on different nodes
     - Pods are preferred to be on a master node.
---
 .../templates/coredns-deployment.yml.j2       | 16 ++++++++++++++
 .../templates/kubedns-autoscaler.yml.j2       | 19 ++++++++++++++++-
 .../ansible/templates/kubedns-deploy.yml.j2   | 21 +++++++++++++++++--
 3 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2
index 5cba6f1f0..dc1f51937 100644
--- a/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2
@@ -34,6 +34,22 @@ spec:
           effect: NoSchedule
         - key: "CriticalAddonsOnly"
           operator: "Exists"
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: "kubernetes.io/hostname"
+            labelSelector:
+              matchLabels:
+                k8s-app: coredns{{ coredns_ordinal_suffix | default('') }}
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: In
+                values:
+                - "true"
       containers:
       - name: coredns
         image: "{{ coredns_image_repo }}:{{ coredns_image_tag }}"
diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2
index d7c30eceb..73ae3a01a 100644
--- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2
@@ -30,7 +30,24 @@ spec:
     spec:
       tolerations:
         - effect: NoSchedule
-          operator: Exists
+          operator: Equal
+          key: node-role.kubernetes.io/master
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: "kubernetes.io/hostname"
+            labelSelector:
+              matchLabels:
+                k8s-app: kubedns-autoscaler
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: In
+                values:
+                - "true"
       containers:
       - name: autoscaler
         image: "{{ kubednsautoscaler_image_repo }}:{{ kubednsautoscaler_image_tag }}"
diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2
index cfce65f0e..bb040780f 100644
--- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2
@@ -30,8 +30,25 @@ spec:
       tolerations:
       - key: "CriticalAddonsOnly"
         operator: "Exists"
-      - effect: NoSchedule
-        operator: Exists
+      - effect: "NoSchedule"
+        operator: "Equal"
+        key: "node-role.kubernetes.io/master"
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: "kubernetes.io/hostname"
+            labelSelector:
+              matchLabels:
+                k8s-app: kube-dns
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: In
+                values:
+                - "true"
       volumes:
       - name: kube-dns-config
         configMap: