From 68fd7e39da3d336f8796a738cd051cb162f86282 Mon Sep 17 00:00:00 2001 From: Thomas Nys Date: Tue, 29 Jan 2019 08:39:27 +0100 Subject: [PATCH] Set cluster DNS correctly in case of nodelocal dns cache (#3879) * Set cluster DNS correctly in case of nodelocal dns cache * Pass in cluster_ip based on dns mode * Disable nodelocaldns by default * Fix syntax error * Fix syntax issue * Add nodelocadns ip to vars of node installation * Change location of nodelocaldns_ip * Try to remove newlines from jinja template * Add debug for config file * Move parameter logic outside of template * Adapt templates after feedback * Remove debugging --- .../group_vars/k8s-cluster/k8s-cluster.yml | 1 + .../kubernetes-apps/ansible/defaults/main.yml | 1 - .../ansible/tasks/nodelocaldns.yml | 26 +++++++++++- .../templates/nodelocaldns-config.yml.j2 | 40 ++++++------------- .../node/templates/kubelet.kubeadm.env.j2 | 3 ++ roles/kubespray-defaults/defaults/main.yaml | 1 + 6 files changed, 41 insertions(+), 31 deletions(-) diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index 02498961c..fdf074f29 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -122,6 +122,7 @@ dns_mode: coredns #manual_dns_server: 10.x.x.x # Enable nodelocal dns cache enable_nodelocaldns: False +nodelocaldns_ip: 169.254.25.10 # Can be docker_dns, host_resolvconf or none resolvconf_mode: docker_dns diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 42d9c7a4d..bf819baa6 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -10,7 +10,6 @@ dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas|int > 1 else 'f # nodelocaldns nodelocaldns_cpu_requests: 100m -nodelocaldns_ip: 169.254.25.10 nodelocaldns_memory_limit: 170Mi nodelocaldnsdns_memory_requests: 70Mi diff --git a/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml b/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml index ef0d61a7f..d43112adf 100644 --- a/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml +++ b/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml @@ -1,4 +1,16 @@ --- +- name: Kubernetes Apps | set up necessary nodelocaldns parameters + set_fact: + clusterIP: >- + {%- if dns_mode in ['kubedns', 'coredns', 'coredns_dual'] -%} + {{ skydns_server }} + {%- elif dns_mode == 'dnsmasq_kubedns' -%} + {{ dnsmasq_dns_server }} + {%- elif dns_mode == 'manual' -%} + {{ manual_dns_server }} + {%- endif -%} + secondaryclusterIP: "{{ skydns_server_secondary }}" + - name: Kubernetes Apps | Lay Down nodelocaldns Template template: src: "{{ item.file }}.j2" @@ -9,8 +21,18 @@ - { name: nodelocaldns, file: nodelocaldns-daemonset.yml, type: daemonset } register: nodelocaldns_manifests vars: - clusterIP: "{{ skydns_server }}" - secondaryclusterIP: "{{ skydns_server_secondary }}" + forwardTarget: >- + {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%} + {{ clusterIP }} {{ secondaryclusterIP }} + {%- else -%} + {{ clusterIP }} + {%- endif -%} + upstreamForwardTarget: >- + {%- if resolvconf_mode == 'host_resolvconf' and upstream_dns_servers is defined and upstream_dns_servers|length > 0 -%} + {{ upstream_dns_servers|join(' ') }} + {%- else -%} + /etc/resolv.conf + {%- endif -%} when: - enable_nodelocaldns == True - inventory_hostname == groups['kube-master'] | first diff --git a/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 index 258289029..ed0c837ef 100644 --- a/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 @@ -14,58 +14,42 @@ data: reload loop bind {{ nodelocaldns_ip }} -{% if secondaryclusterIP is defined and dns_mode == 'coredns_dual' %} - forward . {{ clusterIP }} {{ secondaryclusterIP }} { -{% else %} - forward . {{ clusterIP }} { -{% endif %} - force_tcp + forward . {{ forwardTarget }} { + force_tcp } prometheus :9253 health {{ nodelocaldns_ip }}:8080 - } + } in-addr.arpa:53 { errors cache 30 reload loop bind {{ nodelocaldns_ip }} -{% if secondaryclusterIP is defined %} - forward . {{ clusterIP }} {{ secondaryclusterIP }} { -{% else %} - forward . {{ clusterIP }} { -{% endif %} - force_tcp + forward . {{ forwardTarget }} { + force_tcp } prometheus :9253 - } + } ip6.arpa:53 { errors cache 30 reload loop bind {{ nodelocaldns_ip }} -{% if secondaryclusterIP is defined %} - forward . {{ clusterIP }} {{ secondaryclusterIP }} { -{% else %} - forward . {{ clusterIP }} { -{% endif %} - force_tcp + forward . {{ forwardTarget }} { + force_tcp } prometheus :9253 - } + } .:53 { errors cache 30 reload loop bind {{ nodelocaldns_ip }} -{% if resolvconf_mode == 'host_resolvconf' and upstream_dns_servers is defined and upstream_dns_servers|length > 0 %} - forward . {{ upstream_dns_servers|join(' ') }} { -{% else %} - forward . /etc/resolv.conf { -{% endif %} - force_tcp + forward . {{ upstreamForwardTarget }} { + force_tcp } prometheus :9253 - } + } diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index be0b34b17..80780f2a2 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -76,6 +76,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% else %} {% set kubelet_args_cluster_dns %}{% endset %} {% endif %} +{% if enable_nodelocaldns == True %} +{% set kubelet_args_cluster_dns %}--cluster-dns={{ nodelocaldns_ip }}{% endset %} +{% endif %} {% set kubelet_args_dns %}{{ kubelet_args_cluster_dns }} --cluster-domain={{ dns_domain }} --resolv-conf={{ kube_resolv_conf }}{% endset %} {# Kubelet node labels #} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 05227040d..ed33822ee 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -57,6 +57,7 @@ dns_mode: coredns # Enable nodelocal dns cache enable_nodelocaldns: False +nodelocaldns_ip: 169.254.25.10 # Should be set to a cluster IP if using a custom cluster DNS # manual_dns_server: 10.x.x.x