From e40368ae2bede83978523cb1d4c04866c322468e Mon Sep 17 00:00:00 2001 From: woopstar Date: Tue, 13 Mar 2018 12:00:05 +0100 Subject: [PATCH] Add CoreDNS support with various fixes Added CoreDNS to downloads Updated with labels. Should now work without RBAC too Fix DNS settings on hosts Rename CoreDNS service from kube-dns to coredns Add rotate based on http://edgeofsanity.net/rant/2017/12/20/systemd-resolved-is-broken.html Updated docs with CoreDNS info Added labels and fixed minor settings from official yaml file: https://github.com/kubernetes/kubernetes/blob/release-1.9/cluster/addons/dns/coredns.yaml.sed Added a secondary deployment and secondary service ip. This is to mitigate dns timeouts and create high resitency for failures. See discussion at 'https://github.com/coreos/coreos-kubernetes/issues/641#issuecomment-281174806' Set dns list correct. Thanks to @whereismyjetpack Only download KubeDNS or CoreDNS if selected Move dns cleanup to its own file and import tasks based on dns mode Fix install of KubeDNS when dnsmask_kubedns mode is selected Add new dns option coredns_dual for dual stack deployment. Added variable to configure replicas deployed. Updated docs for dual stack deployment. Removed rotate option in resolv.conf. Run DNS manifests for CoreDNS and KubeDNS Set skydns servers on dual stack deployment Use only one template for CoreDNS dual deployment Set correct cluster ip for the dns server --- docs/dns-stack.md | 8 ++ docs/vars.md | 7 +- inventory/sample/group_vars/k8s-cluster.yml | 8 +- roles/docker/tasks/set_facts_dns.yml | 6 +- roles/download/defaults/main.yml | 17 +++- .../kubernetes-apps/ansible/defaults/main.yml | 3 + .../ansible/tasks/cleanup_dns.yml | 54 +++++++++++++ .../kubernetes-apps/ansible/tasks/coredns.yml | 39 +++++++++ .../kubernetes-apps/ansible/tasks/kubedns.yml | 41 ++++++++++ roles/kubernetes-apps/ansible/tasks/main.yml | 69 ++++------------ .../templates/coredns-clusterrole.yml.j2 | 19 +++++ .../coredns-clusterrolebinding.yml.j2 | 18 +++++ .../ansible/templates/coredns-config.yml.j2 | 22 +++++ .../templates/coredns-deployment.yml.j2 | 81 +++++++++++++++++++ .../ansible/templates/coredns-sa.yml.j2 | 9 +++ .../ansible/templates/coredns-svc.yml.j2 | 22 +++++ .../node/templates/kubelet.kubeadm.env.j2 | 4 +- .../node/templates/kubelet.standard.env.j2 | 4 +- .../preinstall/tasks/set_resolv_facts.yml | 4 +- roles/kubespray-defaults/defaults/main.yaml | 1 + 20 files changed, 369 insertions(+), 67 deletions(-) create mode 100644 roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml create mode 100644 roles/kubernetes-apps/ansible/tasks/coredns.yml create mode 100644 roles/kubernetes-apps/ansible/tasks/kubedns.yml create mode 100644 roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 create mode 100644 roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 create mode 100644 roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 create mode 100644 roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 create mode 100644 roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 create mode 100644 roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 diff --git a/docs/dns-stack.md b/docs/dns-stack.md index 6215114af..1deb88776 100644 --- a/docs/dns-stack.md +++ b/docs/dns-stack.md @@ -62,6 +62,14 @@ other queries are forwardet to the nameservers found in ``upstream_dns_servers`` This does not install the dnsmasq DaemonSet and instructs kubelet to directly use kubedns/skydns for all queries. +#### coredns +This does not install the dnsmasq DaemonSet and instructs kubelet to directly use CoreDNS for +all queries. + +#### coredns_dual +This does not install the dnsmasq DaemonSet and instructs kubelet to directly use CoreDNS for +all queries. It will also deploy a secondary CoreDNS stack + #### manual This does not install dnsmasq or kubedns, but allows you to specify `manual_dns_server`, which will be configured on nodes for handling Pod DNS. diff --git a/docs/vars.md b/docs/vars.md index 3303f6bcb..f612b4f52 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -63,7 +63,8 @@ following default cluster paramters: bits in kube_pods_subnet dictates how many kube-nodes can be in cluster. * *dns_setup* - Enables dnsmasq * *dnsmasq_dns_server* - Cluster IP for dnsmasq (default is 10.233.0.2) -* *skydns_server* - Cluster IP for KubeDNS (default is 10.233.0.3) +* *skydns_server* - Cluster IP for DNS (default is 10.233.0.3) +* *skydns_server_secondary* - Secondary Cluster IP for CoreDNS used with coredns_dual deployment (default is 10.233.0.4) * *cloud_provider* - Enable extra Kubelet option if operating inside GCE or OpenStack (default is unset) * *kube_hostpath_dynamic_provisioner* - Required for use of PetSets type in @@ -105,9 +106,9 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st * *http_proxy/https_proxy/no_proxy* - Proxy variables for deploying behind a proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames that correspond to each node. -* *kubelet_deployment_type* - Controls which platform to deploy kubelet on. +* *kubelet_deployment_type* - Controls which platform to deploy kubelet on. Available options are ``host``, ``rkt``, and ``docker``. ``docker`` mode - is unlikely to work on newer releases. Starting with Kubernetes v1.7 + is unlikely to work on newer releases. Starting with Kubernetes v1.7 series, this now defaults to ``host``. Before v1.7, the default was Docker. This is because of cgroup [issues](https://github.com/kubernetes/kubernetes/issues/43704). * *kubelet_load_modules* - For some things, kubelet needs to load kernel modules. For example, diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 128e8cc99..df5d3513d 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -111,14 +111,17 @@ kube_apiserver_insecure_port: 8080 # (http) # Kube-proxy proxyMode configuration. # Can be ipvs, iptables -kube_proxy_mode: iptables +kube_proxy_mode: iptables + +## Encrypting Secret Data at Rest (experimental) +kube_encrypt_secret_data: false # DNS configuration. # Kubernetes cluster name, also will be used as DNS domain cluster_name: cluster.local # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods ndots: 2 -# Can be dnsmasq_kubedns, kubedns, manual or none +# Can be dnsmasq_kubedns, kubedns, coredns, coredns_dual, manual or none dns_mode: kubedns # Set manual server if using a custom cluster DNS server #manual_dns_server: 10.x.x.x @@ -129,6 +132,7 @@ resolvconf_mode: docker_dns deploy_netchecker: false # Ip address of the kubernetes skydns service skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" +skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}" dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" dns_domain: "{{ cluster_name }}" diff --git a/roles/docker/tasks/set_facts_dns.yml b/roles/docker/tasks/set_facts_dns.yml index 7152b442b..6fe516c2d 100644 --- a/roles/docker/tasks/set_facts_dns.yml +++ b/roles/docker/tasks/set_facts_dns.yml @@ -3,8 +3,10 @@ - name: set dns server for docker set_fact: docker_dns_servers: |- - {%- if dns_mode == 'kubedns' -%} + {%- if dns_mode in ['kubedns', 'coredns'] -%} {{ [ skydns_server ] }} + {%- elif dns_mode == 'coredns_dual' -%} + {{ [ skydns_server ] + [ skydns_server_secondary ] }} {%- elif dns_mode == 'dnsmasq_kubedns' -%} {{ [ dnsmasq_dns_server ] }} {%- elif dns_mode == 'manual' -%} @@ -24,7 +26,7 @@ - name: add upstream dns servers (only when dnsmasq is not used) set_fact: docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers|default([]) }}" - when: dns_mode == 'kubedns' + when: dns_mode in ['kubedns', 'coredns', 'coreos_dual'] - name: add global searchdomains set_fact: diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index d87f4b923..b43a5aa0b 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -100,6 +100,9 @@ dnsmasq_image_tag: "{{ dnsmasq_version }}" kubedns_version: 1.14.8 kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-amd64" kubedns_image_tag: "{{ kubedns_version }}" +coredns_version: 1.1.0 +coredns_image_repo: "docker.io/coredns/coredns" +coredns_image_tag: "{{ coredns_version }}" dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64" dnsmasq_nanny_image_tag: "{{ kubedns_version }}" dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-amd64" @@ -274,25 +277,31 @@ downloads: tag: "{{ dnsmasq_image_tag }}" sha256: "{{ dnsmasq_digest_checksum|default(None) }}" kubedns: - enabled: true + enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true repo: "{{ kubedns_image_repo }}" tag: "{{ kubedns_image_tag }}" sha256: "{{ kubedns_digest_checksum|default(None) }}" + coredns: + enabled: "{{ dns_mode in ['coredns', 'coredns_dual'] }}" + container: true + repo: "{{ coredns_image_repo }}" + tag: "{{ coredns_image_tag }}" + sha256: "{{ coredns_digest_checksum|default(None) }}" dnsmasq_nanny: - enabled: true + enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true repo: "{{ dnsmasq_nanny_image_repo }}" tag: "{{ dnsmasq_nanny_image_tag }}" sha256: "{{ dnsmasq_nanny_digest_checksum|default(None) }}" dnsmasq_sidecar: - enabled: true + enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true repo: "{{ dnsmasq_sidecar_image_repo }}" tag: "{{ dnsmasq_sidecar_image_tag }}" sha256: "{{ dnsmasq_sidecar_digest_checksum|default(None) }}" kubednsautoscaler: - enabled: true + enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true repo: "{{ kubednsautoscaler_image_repo }}" tag: "{{ kubednsautoscaler_image_tag }}" diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 350f663a1..4dc4be212 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -10,6 +10,9 @@ dns_memory_requests: 70Mi kubedns_min_replicas: 2 kubedns_nodes_per_replica: 10 +# CoreDNS +coredns_replicas: 2 + # Images kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-amd64" kubedns_image_tag: "{{ kubedns_version }}" diff --git a/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml b/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml new file mode 100644 index 000000000..5f8356cf9 --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml @@ -0,0 +1,54 @@ +--- +- name: Kubernetes Apps | Delete old CoreDNS resources + kube: + name: "coredns" + namespace: "{{ system_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item }}" + state: absent + with_items: + - 'deploy' + - 'configmap' + - 'svc' + tags: + - upgrade + +- name: Kubernetes Apps | Delete kubeadm CoreDNS + kube: + name: "coredns" + namespace: "{{ system_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "deploy" + state: absent + when: + - kubeadm_enabled|default(false) + - kubeadm_init.changed|default(false) + - inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Apps | Delete old KubeDNS resources + kube: + name: "kube-dns" + namespace: "{{ system_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item }}" + state: absent + with_items: + - 'deploy' + - 'svc' + tags: + - upgrade + +- name: Kubernetes Apps | Delete kubeadm KubeDNS + kube: + name: "kube-dns" + namespace: "{{ system_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item }}" + state: absent + with_items: + - 'deploy' + - 'svc' + when: + - kubeadm_enabled|default(false) + - kubeadm_init.changed|default(false) + - inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ansible/tasks/coredns.yml b/roles/kubernetes-apps/ansible/tasks/coredns.yml new file mode 100644 index 000000000..fcd6c4c6d --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/coredns.yml @@ -0,0 +1,39 @@ +--- +- name: Kubernetes Apps | Lay Down CoreDNS Template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - { name: coredns, file: coredns-config.yml, type: configmap } + - { name: coredns, file: coredns-sa.yml, type: sa } + - { name: coredns, file: coredns-deployment.yml, type: deployment } + - { name: coredns, file: coredns-svc.yml, type: svc } + - { name: coredns, file: coredns-clusterrole.yml, type: clusterrole } + - { name: coredns, file: coredns-clusterrolebinding.yml, type: clusterrolebinding } + register: coredns_manifests + vars: + clusterIP: "{{ skydns_server }}" + when: + - dns_mode in ['coredns', 'coredns_dual'] + - inventory_hostname == groups['kube-master'][0] + - rbac_enabled or item.type not in rbac_resources + tags: + - coredns + +- name: Kubernetes Apps | Lay Down Secondary CoreDNS Template + template: + src: "{{ item.src }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - { name: coredns, src: coredns-deployment.yml, file: coredns-deployment-secondary.yml, type: deployment } + - { name: coredns, src: coredns-svc.yml, file: coredns-svc-secondary.yml, type: svc } + register: coredns_secondary_manifests + vars: + clusterIP: "{{ skydns_server_secondary }}" + coredns_ordinal_suffix: "-secondary" + when: + - dns_mode == 'coredns_dual' + - inventory_hostname == groups['kube-master'][0] + - rbac_enabled or item.type not in rbac_resources + tags: + - coredns diff --git a/roles/kubernetes-apps/ansible/tasks/kubedns.yml b/roles/kubernetes-apps/ansible/tasks/kubedns.yml new file mode 100644 index 000000000..c4c34ecf8 --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/kubedns.yml @@ -0,0 +1,41 @@ +--- + +- name: Kubernetes Apps | Lay Down KubeDNS Template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - { name: kube-dns, file: kubedns-sa.yml, type: sa } + - { name: kube-dns, file: kubedns-deploy.yml, type: deployment } + - { name: kube-dns, file: kubedns-svc.yml, type: svc } + - { name: kubedns-autoscaler, file: kubedns-autoscaler-sa.yml, type: sa } + - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrole.yml, type: clusterrole } + - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding } + - { name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment } + register: kubedns_manifests + when: + - dns_mode in ['kubedns','dnsmasq_kubedns'] + - inventory_hostname == groups['kube-master'][0] + - rbac_enabled or item.type not in rbac_resources + tags: + - dnsmasq + +# see https://github.com/kubernetes/kubernetes/issues/45084, only needed for "old" kube-dns +- name: Kubernetes Apps | Patch system:kube-dns ClusterRole + command: > + {{ bin_dir }}/kubectl patch clusterrole system:kube-dns + --patch='{ + "rules": [ + { + "apiGroups" : [""], + "resources" : ["endpoints", "services"], + "verbs": ["list", "watch", "get"] + } + ] + }' + when: + - dns_mode in ['kubedns', 'dnsmasq_kubedns'] + - inventory_hostname == groups['kube-master'][0] + - rbac_enabled and kubedns_version|version_compare("1.11.0", "<", strict=True) + tags: + - dnsmasq diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index a25d595eb..55d417982 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -11,66 +11,26 @@ delay: 2 when: inventory_hostname == groups['kube-master'][0] -- name: Kubernetes Apps | Delete old kubedns resources - kube: - name: "kubedns" - namespace: "{{ system_namespace }}" - kubectl: "{{ bin_dir }}/kubectl" - resource: "{{ item }}" - state: absent - with_items: - - 'deploy' - - 'svc' +- name: Kubernetes Apps | Cleanup DNS + import_tasks: tasks/cleanup_dns.yml + when: + - inventory_hostname == groups['kube-master'][0] tags: - upgrade -- name: Kubernetes Apps | Delete kubeadm kubedns - kube: - name: "kubedns" - namespace: "{{ system_namespace }}" - kubectl: "{{ bin_dir }}/kubectl" - resource: "deploy" - state: absent +- name: Kubernetes Apps | CoreDNS + import_tasks: "tasks/coredns.yml" when: - - kubeadm_enabled|default(false) - - kubeadm_init.changed|default(false) + - dns_mode in ['coredns', 'coredns_dual'] - inventory_hostname == groups['kube-master'][0] - -- name: Kubernetes Apps | Lay Down KubeDNS Template - template: - src: "{{ item.file }}.j2" - dest: "{{ kube_config_dir }}/{{ item.file }}" - with_items: - - { name: kube-dns, file: kubedns-sa.yml, type: sa } - - { name: kube-dns, file: kubedns-deploy.yml, type: deployment } - - { name: kube-dns, file: kubedns-svc.yml, type: svc } - - { name: kubedns-autoscaler, file: kubedns-autoscaler-sa.yml, type: sa } - - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrole.yml, type: clusterrole } - - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding } - - { name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment } - register: manifests - when: - - dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] - - rbac_enabled or item.type not in rbac_resources tags: - - dnsmasq + - coredns -# see https://github.com/kubernetes/kubernetes/issues/45084, only needed for "old" kube-dns -- name: Kubernetes Apps | Patch system:kube-dns ClusterRole - command: > - {{ bin_dir }}/kubectl patch clusterrole system:kube-dns - --patch='{ - "rules": [ - { - "apiGroups" : [""], - "resources" : ["endpoints", "services"], - "verbs": ["list", "watch", "get"] - } - ] - }' +- name: Kubernetes Apps | KubeDNS + import_tasks: "tasks/kubedns.yml" when: - - dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] - - rbac_enabled and kubedns_version|version_compare("1.11.0", "<", strict=True) + - dns_mode in ['kubedns', 'dnsmasq_kubedns'] + - inventory_hostname == groups['kube-master'][0] tags: - dnsmasq @@ -82,7 +42,10 @@ resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" - with_items: "{{ manifests.results }}" + with_items: + - "{{ kubedns_manifests.results | default({}) }}" + - "{{ coredns_manifests.results | default({}) }}" + - "{{ coredns_secondary_manifests.results | default({}) }}" when: - dns_mode != 'none' - inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 new file mode 100644 index 000000000..4136d603e --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 @@ -0,0 +1,19 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: Reconcile + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch diff --git a/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 new file mode 100644 index 000000000..6c49d047f --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 @@ -0,0 +1,18 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: EnsureExists + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: {{ system_namespace }} diff --git a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 new file mode 100644 index 000000000..983d2579f --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: {{ system_namespace }} + labels: + addonmanager.kubernetes.io/mode: EnsureExists +data: + Corefile: | + .:53 { + errors + health + kubernetes {{ cluster_name }} in-addr.arpa ip6.arpa { + pods insecure + upstream /etc/resolv.conf + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + proxy . /etc/resolv.conf + cache 30 + } diff --git a/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 new file mode 100644 index 000000000..30128d566 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 @@ -0,0 +1,81 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: coredns{{ coredns_ordinal_suffix | default('') }} + namespace: {{ system_namespace }} + labels: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" +spec: + replicas: {{ coredns_replicas }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 10% + selector: + matchLabels: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + template: + metadata: + labels: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: +{% if rbac_enabled %} + serviceAccountName: coredns +{% endif %} + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: coredns + image: "{{ coredns_image_repo }}:{{ coredns_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + memory: {{ dns_memory_limit }} + requests: + cpu: {{ dns_cpu_requests }} + memory: {{ dns_memory_requests }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile diff --git a/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 new file mode 100644 index 000000000..db5682354 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: {{ system_namespace }} + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 new file mode 100644 index 000000000..c5b76b0b5 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns{{ coredns_ordinal_suffix | default('') }} + namespace: {{ system_namespace }} + labels: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" +spec: + selector: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + clusterIP: {{ clusterIP }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index c8cf40e7b..28467a501 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -50,8 +50,10 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {# DNS settings for kubelet #} -{% if dns_mode == 'kubedns' %} +{% if dns_mode in ['kubedns', 'coredns'] %} {% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }}{% endset %} +{% elif dns_mode == 'coredns_dual' %} +{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }},{{ skydns_server_secondary }}{% endset %} {% elif dns_mode == 'dnsmasq_kubedns' %} {% set kubelet_args_cluster_dns %}--cluster-dns={{ dnsmasq_dns_server }}{% endset %} {% elif dns_mode == 'manual' %} diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index 8e05e0253..d33adfba7 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -42,8 +42,10 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --enforce-node-allocatable={{ kubelet_enforce_node_allocatable }} {% endif %}{% endset %} {# DNS settings for kubelet #} -{% if dns_mode == 'kubedns' %} +{% if dns_mode in ['kubedns', 'coredns'] %} {% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }}{% endset %} +{% elif dns_mode == 'coredns_dual' %} +{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }},{{ skydns_server_secondary }}{% endset %} {% elif dns_mode == 'dnsmasq_kubedns' %} {% set kubelet_args_cluster_dns %}--cluster-dns={{ dnsmasq_dns_server }}{% endset %} {% elif dns_mode == 'manual' %} diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml index fdc46125e..eb8f3f43f 100644 --- a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml @@ -93,8 +93,10 @@ - name: pick dnsmasq cluster IP or default resolver set_fact: dnsmasq_server: |- - {%- if dns_mode == 'kubedns' and not dns_early|bool -%} + {%- if dns_mode in ['kubedns', 'coredns'] and not dns_early|bool -%} {{ [ skydns_server ] + upstream_dns_servers|default([]) }} + {%- elif dns_mode == 'coredns_dual' and not dns_early|bool -%} + {{ [ skydns_server ] + [ skydns_server_secondary ] + upstream_dns_servers|default([]) }} {%- elif dns_mode == 'manual' and not dns_early|bool -%} {{ [ manual_dns_server ] + upstream_dns_servers|default([]) }} {%- elif dns_early|bool -%} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 61f11e97f..0b10adf62 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -49,6 +49,7 @@ resolvconf_mode: docker_dns deploy_netchecker: false # Ip address of the kubernetes skydns service skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" +skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}" dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" dns_domain: "{{ cluster_name }}"