diff --git a/docs/dns-stack.md b/docs/dns-stack.md new file mode 100644 index 000000000..e56ee2fda --- /dev/null +++ b/docs/dns-stack.md @@ -0,0 +1,21 @@ +K8s DNS stack by Kargo +====================== + +Here is an approximate picture of how DNS things working and +being configured by Kargo ansible playbooks: + +![Image](figures/dns.png?raw=true) + +Note that an additional dnsmasq daemon set is installed by Kargo +by default. Kubelet will configure DNS base of all pods to use that +dnsmasq cluster IP. You can disable it with the ``skip_dnsmasq`` +var. This may be the case, if you're fine with Linux limit of max 3 +nameservers in the ``/etc/resolv.conf``. When skipped and bypassed +directly to Kubedns's dnsmasq cluster IP, it greatly simplifies things +by the price of limited nameservers though. + +Nameservers are configured in the hosts' ``/etc/resolv.conf`` files +from the ``nameservers`` (see also ``searchdomains``) vars. While the +``upstream_dns_servers`` will define additional DNS servers for the +dnsmasq daemon set running on all hosts (unless bypassed with +``skip_dnsmasq``). diff --git a/docs/figures/dns.png b/docs/figures/dns.png new file mode 100644 index 000000000..dbdb0c071 Binary files /dev/null and b/docs/figures/dns.png differ diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 2de01828c..90e0196c5 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -103,6 +103,8 @@ kube_apiserver_insecure_port: 8080 # (http) # You still must manually configure all your containers to use this DNS server, # Kubernetes won't do this for you (yet). +# Do not install additional dnsmasq +skip_dnsmasq: false # Upstream dns servers used by dnsmasq upstream_dns_servers: - 8.8.8.8 diff --git a/roles/dnsmasq/tasks/dnsmasq.yml b/roles/dnsmasq/tasks/dnsmasq.yml new file mode 100644 index 000000000..bc37580de --- /dev/null +++ b/roles/dnsmasq/tasks/dnsmasq.yml @@ -0,0 +1,58 @@ +--- +- name: ensure dnsmasq.d directory exists + file: + path: /etc/dnsmasq.d + state: directory + +- name: ensure dnsmasq.d-available directory exists + file: + path: /etc/dnsmasq.d-available + state: directory + +- name: Write dnsmasq configuration + template: + src: 01-kube-dns.conf.j2 + dest: /etc/dnsmasq.d-available/01-kube-dns.conf + mode: 0755 + backup: yes + +- name: Stat dnsmasq configuration + stat: path=/etc/dnsmasq.d/01-kube-dns.conf + register: sym + +- name: Move previous configuration + command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak + changed_when: False + when: sym.stat.islnk is defined and sym.stat.islnk == False + +- name: Enable dnsmasq configuration + file: + src: /etc/dnsmasq.d-available/01-kube-dns.conf + dest: /etc/dnsmasq.d/01-kube-dns.conf + state: link + +- name: Create dnsmasq manifests + template: src={{item.file}} dest=/etc/kubernetes/{{item.file}} + with_items: + - {file: dnsmasq-ds.yml, type: ds} + - {file: dnsmasq-svc.yml, type: svc} + register: manifests + when: inventory_hostname == groups['kube-master'][0] + +- name: Start Resources + kube: + name: dnsmasq + namespace: kube-system + kubectl: "{{bin_dir}}/kubectl" + resource: "{{item.item.type}}" + filename: /etc/kubernetes/{{item.item.file}} + state: "{{item.changed | ternary('latest','present') }}" + with_items: "{{ manifests.results }}" + when: inventory_hostname == groups['kube-master'][0] + +- name: Check for dnsmasq port (pulling image and running container) + wait_for: + host: "{{dns_server}}" + port: 53 + delay: 5 + when: inventory_hostname == groups['kube-node'][0] diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml index f633485d2..46c1604f6 100644 --- a/roles/dnsmasq/tasks/main.yml +++ b/roles/dnsmasq/tasks/main.yml @@ -1,134 +1,5 @@ --- -- name: ensure dnsmasq.d directory exists - file: - path: /etc/dnsmasq.d - state: directory +- include: dnsmasq.yml + when: "{{ not skip_dnsmasq|bool }}" -- name: ensure dnsmasq.d-available directory exists - file: - path: /etc/dnsmasq.d-available - state: directory - -- name: Write dnsmasq configuration - template: - src: 01-kube-dns.conf.j2 - dest: /etc/dnsmasq.d-available/01-kube-dns.conf - mode: 0755 - backup: yes - -- name: Stat dnsmasq configuration - stat: path=/etc/dnsmasq.d/01-kube-dns.conf - register: sym - -- name: Move previous configuration - command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak - changed_when: False - when: sym.stat.islnk is defined and sym.stat.islnk == False - -- name: Enable dnsmasq configuration - file: - src: /etc/dnsmasq.d-available/01-kube-dns.conf - dest: /etc/dnsmasq.d/01-kube-dns.conf - state: link - -- name: Create dnsmasq manifests - template: src={{item.file}} dest=/etc/kubernetes/{{item.file}} - with_items: - - {file: dnsmasq-ds.yml, type: ds} - - {file: dnsmasq-svc.yml, type: svc} - register: manifests - when: inventory_hostname == groups['kube-master'][0] - -- name: Start Resources - kube: - name: dnsmasq - namespace: kube-system - kubectl: "{{bin_dir}}/kubectl" - resource: "{{item.item.type}}" - filename: /etc/kubernetes/{{item.item.file}} - state: "{{item.changed | ternary('latest','present') }}" - with_items: "{{ manifests.results }}" - when: inventory_hostname == groups['kube-master'][0] - -- name: Check for dnsmasq port (pulling image and running container) - wait_for: - host: "{{dns_server}}" - port: 53 - delay: 5 - when: inventory_hostname == groups['kube-node'][0] - - -- name: check resolvconf - shell: which resolvconf - register: resolvconf - ignore_errors: yes - -- name: target resolv.conf file - set_fact: - resolvconffile: >- - {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%} - -- name: generate search domains to resolvconf - set_fact: - searchentries="{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}" - -- name: generate nameservers to resolvconf - set_fact: - nameserverentries="{{ nameservers|default([]) + [ dns_server ] }}" - -- name: Add search resolv.conf - lineinfile: - line: "search {{searchentries}}" - dest: "{{resolvconffile}}" - state: present - insertbefore: BOF - backup: yes - follow: yes - -- name: Add local dnsmasq to resolv.conf - blockinfile: - dest: "{{resolvconffile}}" - block: |- - {% for item in nameserverentries -%} - nameserver {{ item }} - {% endfor %} - state: present - create: yes - backup: yes - follow: yes - marker: "# Ansible nameservers {mark}" - -- name: Add options to resolv.conf - lineinfile: - line: options {{ item }} - dest: "{{resolvconffile}}" - state: present - regexp: "^options.*{{ item }}$" - insertafter: EOF - backup: yes - follow: yes - with_items: - - timeout:2 - - attempts:2 - -- name: Remove search and nameserver options from resolvconf base - lineinfile: - dest: /etc/resolvconf/resolv.conf.d/base - state: absent - regexp: "^{{ item }}.*$" - backup: yes - follow: yes - with_items: - - search - - nameserver - when: resolvconf.rc == 0 - -- name: disable resolv.conf modification by dhclient - copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755 - notify: Dnsmasq | restart network - when: ansible_os_family == "Debian" - -- name: disable resolv.conf modification by dhclient - copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x - notify: Dnsmasq | restart network - when: ansible_os_family == "RedHat" +- include: resolvconf.yml diff --git a/roles/dnsmasq/tasks/resolvconf.yml b/roles/dnsmasq/tasks/resolvconf.yml new file mode 100644 index 000000000..22c135262 --- /dev/null +++ b/roles/dnsmasq/tasks/resolvconf.yml @@ -0,0 +1,100 @@ +--- +- name: check resolvconf + shell: which resolvconf + register: resolvconf + ignore_errors: yes + +- name: target resolv.conf file + set_fact: + resolvconffile: >- + {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%} + +- name: generate search domains to resolvconf + set_fact: + searchentries: + "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}" + +- name: pick dnsmasq cluster IP + set_fact: + dnsmasq_server: >- + {%- if skip_dnsmasq|bool -%}{{ [ skydns_server ] + upstream_dns_servers|default([]) }}{%- else -%}{{ [ dns_server ] }}{%- endif -%} + +- name: generate nameservers to resolvconf + set_fact: + nameserverentries: + "{{ nameservers|default([]) + dnsmasq_server|default([]) }}" + +- name: Remove search and nameserver options from resolvconf head + lineinfile: + dest: /etc/resolvconf/resolv.conf.d/head + state: absent + regexp: "^{{ item }}.*$" + backup: yes + follow: yes + with_items: + - search + - nameserver + when: resolvconf.rc == 0 + notify: Dnsmasq | update resolvconf + +- name: Add search resolv.conf + lineinfile: + line: "search {{searchentries}}" + dest: "{{resolvconffile}}" + state: present + insertbefore: BOF + backup: yes + follow: yes + notify: Dnsmasq | update resolvconf + +- name: Add local dnsmasq to resolv.conf + blockinfile: + dest: "{{resolvconffile}}" + block: |- + {% for item in nameserverentries -%} + nameserver {{ item }} + {% endfor %} + state: present + insertafter: "^search.*$" + create: yes + backup: yes + follow: yes + marker: "# Ansible nameservers {mark}" + notify: Dnsmasq | update resolvconf + +- name: Add options to resolv.conf + lineinfile: + line: options {{ item }} + dest: "{{resolvconffile}}" + state: present + regexp: "^options.*{{ item }}$" + insertafter: EOF + backup: yes + follow: yes + with_items: + - timeout:2 + - attempts:2 + notify: Dnsmasq | update resolvconf + +- name: Remove search and nameserver options from resolvconf base + lineinfile: + dest: /etc/resolvconf/resolv.conf.d/base + state: absent + regexp: "^{{ item }}.*$" + backup: yes + follow: yes + with_items: + - search + - nameserver + when: resolvconf.rc == 0 + notify: Dnsmasq | update resolvconf + +- name: disable resolv.conf modification by dhclient + copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755 + notify: Dnsmasq | restart network + when: ansible_os_family == "Debian" + +- name: disable resolv.conf modification by dhclient + copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x + notify: Dnsmasq | restart network + when: ansible_os_family == "RedHat" diff --git a/roles/dnsmasq/templates/01-kube-dns.conf.j2 b/roles/dnsmasq/templates/01-kube-dns.conf.j2 index a9ef6200e..7d5b77f1e 100644 --- a/roles/dnsmasq/templates/01-kube-dns.conf.j2 +++ b/roles/dnsmasq/templates/01-kube-dns.conf.j2 @@ -4,6 +4,10 @@ listen-address=0.0.0.0 addn-hosts=/etc/hosts +strict-order +# Forward k8s domain to kube-dns +server=/{{ dns_domain }}/{{ skydns_server }} + #Set upstream dns servers {% if upstream_dns_servers is defined %} {% for srv in upstream_dns_servers %} @@ -14,8 +18,9 @@ server={{ srv }} server=8.8.4.4 {% endif %} -# Forward k8s domain to kube-dns -server=/{{ dns_domain }}/{{ skydns_server }} - -# Forward reverse lookups for k8s service addresses to kube-dns -rev-server={{ kube_service_addresses }},{{ skydns_server }} +no-resolv +no-negcache +cache-size=1000 +max-cache-ttl=10 +max-ttl=20 +log-facility=- diff --git a/roles/dnsmasq/templates/dnsmasq-ds.yml b/roles/dnsmasq/templates/dnsmasq-ds.yml index 7e5d3dffa..f1f622bbd 100644 --- a/roles/dnsmasq/templates/dnsmasq-ds.yml +++ b/roles/dnsmasq/templates/dnsmasq-ds.yml @@ -50,3 +50,4 @@ spec: - name: etcdnsmasqdavailable hostPath: path: /etc/dnsmasq.d-available + dnsPolicy: Default # Don't use cluster DNS. diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml b/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml index 8e6000167..e58639deb 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml +++ b/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml @@ -65,6 +65,7 @@ spec: - name: dnsmasq image: gcr.io/google_containers/kube-dnsmasq-amd64:1.3 args: + - --log-facility=- - --cache-size=1000 - --no-resolv - --server=127.0.0.1#10053 diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 7dc7d1183..94da756be 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -8,13 +8,6 @@ kube_resolv_conf: "/etc/resolv.conf" kube_proxy_mode: iptables -# IP address of the DNS server. -# Kubernetes will create a pod with several containers, serving as the DNS -# server and expose it under this IP address. The IP address must be from -# the range specified as kube_service_addresses. This magic will actually -# pick the 10th ip address in the kube_service_addresses range and use that. -dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}" - # kube_api_runtime_config: # - extensions/v1beta1/daemonsets=true # - extensions/v1beta1/deployments=true diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index c7d20c420..f55feefa9 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -19,7 +19,9 @@ KUBELET_HOSTNAME="--hostname-override={{ inventory_hostname }}" KUBELET_REGISTER_NODE="--register-node=false" {% endif %} # location of the api-server -{% if dns_setup %} +{% if dns_setup|bool and skip_dnsmasq|bool %} +KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}" +{% elif dns_setup|bool %} KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}" {% else %} KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"