Rework DNS stack to meet hostnet pods needs

* For Debian/RedHat OS families (with NetworkManager/dhclient/resolvconf
  optionally enabled) prepend /etc/resolv.conf with required nameservers,
  options, and supersede domain and search domains via the dhclient/resolvconf
  hooks.

* Drop (z)nodnsupdate dhclient hook and re-implement it to complement the
  resolvconf -u command, which is distro/cloud provider specific.
  Update docs as well.

* Enable network restart to apply and persist changes and simplify handlers
  to rely on network restart only. This fixes DNS resolve for hostnet K8s
  pods for Red Hat OS family. Skip network restart for canal/calico plugins,
  unless https://github.com/projectcalico/felix/issues/1185 fixed.

* Replace linefiles line plus with_items to block mode as it's faster.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
Co-authored-by: Matthew Mosesohn <mmosesohn@mirantis.com>
pull/667/head
Bogdan Dobrelya 2016-11-30 14:06:11 +01:00
parent e5ad0836bc
commit 3117858dcd
8 changed files with 185 additions and 138 deletions

View File

@ -38,6 +38,17 @@ or `8.8.8.8`. And domain is set to the default ``dns_domain`` value as well.
Later, the nameservers will be reconfigured to the DNS service IP that Kargo Later, the nameservers will be reconfigured to the DNS service IP that Kargo
configures for K8s cluster. configures for K8s cluster.
Also note, existing records will be purged from the `/etc/resolv.conf`,
including base/head/cloud-init config files and those that come from dhclient.
This is required for hostnet pods networking and for [kubelet to not exceed search domains
limits](https://github.com/kubernetes/kubernetes/issues/9229).
New search, nameserver records and options will be defined from the aforementioned vars:
* Via resolvconf's head file, if resolvconf installed.
* Via dhclient's DNS update hook.
* Via cloud-init (CoreOS only).
* Statically in the `/etc/resolv.conf`, if none of above is applicable.
DNS configuration details DNS configuration details
------------------------- -------------------------

View File

@ -2,9 +2,10 @@
command: /bin/true command: /bin/true
notify: notify:
- Preinstall | reload network - Preinstall | reload network
- Preinstall | update resolvconf - Preinstall | reload kubelet
when: ansible_os_family != "CoreOS" when: ansible_os_family != "CoreOS"
# FIXME(bogdando) https://github.com/projectcalico/felix/issues/1185
- name: Preinstall | reload network - name: Preinstall | reload network
service: service:
name: >- name: >-
@ -14,14 +15,7 @@
networking networking
{%- endif %} {%- endif %}
state: restarted state: restarted
when: ansible_os_family != "RedHat" and ansible_os_family != "CoreOS" when: ansible_os_family != "CoreOS" or kube_network_plugin not in ['canal', 'calico']
- name: Preinstall | update resolvconf
command: /bin/true
notify:
- Preinstall | reload resolvconf
- Preinstall | reload kubelet
when: ansible_os_family != "CoreOS"
- name: Preinstall | update resolvconf for CoreOS - name: Preinstall | update resolvconf for CoreOS
command: /bin/true command: /bin/true
@ -30,10 +24,6 @@
- Preinstall | reload kubelet - Preinstall | reload kubelet
when: ansible_os_family == "CoreOS" when: ansible_os_family == "CoreOS"
- name: Preinstall | reload resolvconf
command: /sbin/resolvconf -u
ignore_errors: true
- name: Preinstall | apply resolvconf cloud-init - name: Preinstall | apply resolvconf cloud-init
command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }} command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
when: ansible_os_family == "CoreOS" when: ansible_os_family == "CoreOS"

View File

@ -0,0 +1,33 @@
---
- name: Configure dhclient to prepend nameservers and supersede search/domain
blockinfile:
block: |-
{% for item in [ supersede_domain, supersede_search, prepend_nameserver ] -%}
{{ item }}
{% endfor %}
dest: "{{dhclientconffile}}"
create: yes
state: present
insertbefore: BOF
backup: yes
follow: yes
marker: "# Ansible entries {mark}"
notify: Preinstall | restart network
- name: Configue dhclient hooks for resolv.conf (non-RH)
template:
src: dhclient_dnsupdate.sh.j2
dest: "{{ dhclienthookfile }}"
owner: root
mode: 0755
notify: Preinstall | restart network
when: ansible_os_family != "RedHat"
- name: Configue dhclient hooks for resolv.conf (RH-only)
template:
src: dhclient_dnsupdate_rh.sh.j2
dest: "{{ dhclienthookfile }}"
owner: root
mode: 0755
notify: Preinstall | restart network
when: ansible_os_family == "RedHat"

View File

@ -1,120 +1,34 @@
--- ---
- name: check resolvconf
shell: which resolvconf
register: resolvconf
ignore_errors: yes
changed_when: false
tags: facts
- name: check kubelet
stat:
path: "{{ bin_dir }}/kubelet"
register: kubelet
changed_when: false
tags: facts
- name: check if early DNS configuration stage
set_fact:
dns_early: >-
{%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
tags: facts
- name: target resolv.conf file
set_fact:
resolvconffile: >-
{%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
when: ansible_os_family != "CoreOS"
tags: facts
- name: target temporary resolvconf cloud init file
set_fact:
resolvconffile: /tmp/resolveconf_cloud_init_conf
when: ansible_os_family == "CoreOS"
tags: facts
- name: create temporary resolveconf cloud init file - name: create temporary resolveconf cloud init file
command: cp -f /etc/resolv.conf "{{ resolvconffile }}" command: cp -f /etc/resolv.conf "{{ resolvconffile }}"
when: ansible_os_family == "CoreOS" when: ansible_os_family == "CoreOS"
- name: generate search domains to resolvconf - name: Remove search/domain/nameserver options
set_fact:
searchentries:
"{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
tags: facts
- name: decide on dns server IP
set_fact:
dns_server_real: >-
{%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
- name: pick dnsmasq cluster IP or default resolver
set_fact:
dnsmasq_server: |-
{%- if skip_dnsmasq|bool and not dns_early|bool -%}
{{ [ skydns_server ] + upstream_dns_servers|default([]) }}
{%- elif dns_early|bool -%}
{{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
{%- else -%}
{{ [ dns_server ] }}
{%- endif -%}
tags: facts
- name: generate nameservers to resolvconf
set_fact:
nameserverentries:
"{{ dnsmasq_server|default([]) + nameservers|default([]) }}"
tags: facts
- name: Remove search and nameserver options from resolvconf head
lineinfile: lineinfile:
dest: /etc/resolvconf/resolv.conf.d/head dest: "{{item[0]}}"
state: absent state: absent
regexp: "^{{ item }}.*$" regexp: "^{{ item[1] }}.*$"
backup: yes backup: yes
follow: yes follow: yes
with_items: with_nested:
- search - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}"
- nameserver - [ 'search ', 'nameserver ', 'domain ', 'options ' ]
when: resolvconf.rc == 0 notify: Preinstall | restart network
notify: Preinstall | update resolvconf
- name: Remove search and nameserver options from resolvconf cloud init temporary file - name: Add domain/search/nameservers to resolv.conf
lineinfile:
dest: "{{resolvconffile}}"
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: ansible_os_family == "CoreOS"
notify: Preinstall | update resolvconf for CoreOS
- name: Add search domains to resolvconf file
lineinfile:
line: "search {{searchentries}}"
dest: "{{resolvconffile}}"
state: present
insertbefore: BOF
backup: yes
follow: yes
notify: Preinstall | update resolvconf
- name: Add nameservers to resolv.conf
blockinfile: blockinfile:
dest: "{{resolvconffile}}" dest: "{{resolvconffile}}"
block: |- block: |-
{% for item in nameserverentries -%} {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%}
nameserver {{ item }} {{ item }}
{% endfor %} {% endfor %}
state: present state: present
insertafter: "^search default.svc.*$" insertbefore: BOF
create: yes create: yes
backup: yes backup: yes
follow: yes follow: yes
marker: "# Ansible nameservers {mark}" marker: "# Ansible entries {mark}"
notify: Preinstall | update resolvconf notify: Preinstall | restart network
- name: Add options to resolv.conf - name: Add options to resolv.conf
lineinfile: lineinfile:
@ -129,30 +43,7 @@
- ndots:{{ ndots }} - ndots:{{ ndots }}
- timeout:2 - timeout:2
- attempts:2 - attempts:2
notify: Preinstall | update resolvconf
- name: Remove search and nameserver options from resolvconf base
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/base
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
notify: Preinstall | update resolvconf
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
notify: Preinstall | restart network notify: Preinstall | restart network
when: ansible_os_family == "Debian"
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
notify: Preinstall | restart network
when: ansible_os_family == "RedHat"
- name: get temporary resolveconf cloud init file content - name: get temporary resolveconf cloud init file content
command: cat {{ resolvconffile }} command: cat {{ resolvconffile }}
@ -167,3 +58,7 @@
mode: 0644 mode: 0644
notify: Preinstall | update resolvconf for CoreOS notify: Preinstall | update resolvconf for CoreOS
when: ansible_os_family == "CoreOS" when: ansible_os_family == "CoreOS"
- include: dhclient-hooks.yml
when: ansible_os_family != "CoreOS"
tags: [bootstrap-os, resolvconf]

View File

@ -49,6 +49,6 @@
etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=") etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=")
- set_fact: - set_fact:
etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}"
- set_fact:
default_resolver: >- - include: set_resolv_facts.yml
{%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%} tags: [bootstrap-os, resolvconf, facts]

View File

@ -0,0 +1,88 @@
---
- name: check resolvconf
shell: which resolvconf
register: resolvconf
ignore_errors: yes
changed_when: false
- set_fact:
resolvconf: >-
{%- if resolvconf.rc == 0 -%}true{%- else -%}false{%- endif -%}
- set_fact:
private_domains: |-
{% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%}
{{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./
{%- endfor %}
default_resolver: >-
{%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}
- name: check kubelet
stat:
path: "{{ bin_dir }}/kubelet"
register: kubelet
changed_when: false
- name: check if early DNS configuration stage
set_fact:
dns_early: >-
{%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
- name: target resolv.conf files
set_fact:
resolvconffile: /etc/resolv.conf
base: >-
{%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}
head: >-
{%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}
when: ansible_os_family != "CoreOS"
- name: target temporary resolvconf cloud init file (CoreOS)
set_fact: resolvconffile=/tmp/resolveconf_cloud_init_conf
when: ansible_os_family == "CoreOS"
- name: target dhclient conf/hook files for Red Hat family
set_fact:
dhclientconffile: /etc/dhclient.conf
dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh
when: ansible_os_family == "RedHat"
- name: target dhclient conf/hook files for Debian family
set_fact:
dhclientconffile: /etc/dhcp/dhclient.conf
dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
when: ansible_os_family == "Debian"
- name: generate search domains to resolvconf
set_fact:
searchentries:
search {{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}
domainentry:
domain {{ dns_domain }}
supersede_search:
supersede domain-search "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join('", "') }}";
supersede_domain:
supersede domain-name "{{ dns_domain }}";
- name: decide on dns server IP
set_fact:
dns_server_real: >-
{%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
- name: pick dnsmasq cluster IP or default resolver
set_fact:
dnsmasq_server: |-
{%- if skip_dnsmasq|bool and not dns_early|bool -%}
{{ [ skydns_server ] + upstream_dns_servers|default([]) }}
{%- elif dns_early|bool -%}
{{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
{%- else -%}
{{ [ dns_server ] }}
{%- endif -%}
- name: generate nameservers to resolvconf
set_fact:
nameserverentries:
nameserver {{( dnsmasq_server|default([]) + nameservers|default([])) | join(',nameserver ')}}
prepend_nameserver:
prepend domain-name-servers {{( dnsmasq_server|default([]) + nameservers|default([])) | join(', ') }};

View File

@ -0,0 +1,13 @@
#!/bin/sh
#
# Prepend resolver options to /etc/resolv.conf after dhclient`
# regenerates the file. See man (5) resolver for more details.
#
if [ $reason = "BOUND" ]; then
if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
RESOLV_CONF=$(cat /etc/resolv.conf)
OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
fi
fi

View File

@ -0,0 +1,17 @@
#!/bin/sh
#
# Prepend resolver options to /etc/resolv.conf after dhclient`
# regenerates the file. See man (5) resolver for more details.
#
zdnsupdate_config() {
if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
RESOLV_CONF=$(cat /etc/resolv.conf)
OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
fi
}
zdnsupdate_restore() {
:
}