Rework DNS stack to meet hostnet pods needs

* For Debian/RedHat OS families (with NetworkManager/dhclient/resolvconf
  optionally enabled) prepend /etc/resolv.conf with required nameservers,
  options, and supersede domain and search domains via the dhclient/resolvconf
  hooks.

* Drop (z)nodnsupdate dhclient hook and re-implement it to complement the
  resolvconf -u command, which is distro/cloud provider specific.
  Update docs as well.

* Enable network restart to apply and persist changes and simplify handlers
  to rely on network restart only. This fixes DNS resolve for hostnet K8s
  pods for Red Hat OS family. Skip network restart for canal/calico plugins,
  unless https://github.com/projectcalico/felix/issues/1185 fixed.

* Replace linefiles line plus with_items to block mode as it's faster.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
Co-authored-by: Matthew Mosesohn <mmosesohn@mirantis.com>

docs/dns-stack.md

@@ -38,6 +38,17 @@ or `8.8.8.8`. And domain is set to the default ``dns_domain`` value as well.
 Later, the nameservers will be reconfigured to the DNS service IP that Kargo
 configures for K8s cluster.
 
+Also note, existing records will be purged from the `/etc/resolv.conf`,
+including base/head/cloud-init config files and those that come from dhclient.
+This is required for hostnet pods networking and for [kubelet to not exceed search domains
+limits](https://github.com/kubernetes/kubernetes/issues/9229).
+
+New search, nameserver records and options will be defined from the aforementioned vars:
+* Via resolvconf's head file, if resolvconf installed.
+* Via dhclient's DNS update hook.
+* Via cloud-init (CoreOS only).
+* Statically in the `/etc/resolv.conf`, if none of above is applicable.
+
 DNS configuration details
 -------------------------
 

roles/kubernetes/preinstall/handlers/main.yml

@@ -2,9 +2,10 @@
   command: /bin/true
   notify:
     - Preinstall | reload network
-    - Preinstall | update resolvconf
+    - Preinstall | reload kubelet
   when: ansible_os_family != "CoreOS"
 
+  # FIXME(bogdando) https://github.com/projectcalico/felix/issues/1185
 - name: Preinstall | reload network
   service:
     name: >-
@@ -14,14 +15,7 @@
       networking
       {%- endif %}
     state: restarted
-  when: ansible_os_family != "RedHat" and ansible_os_family != "CoreOS"
-
-- name: Preinstall | update resolvconf
-  command: /bin/true
-  notify:
-    - Preinstall | reload resolvconf
-    - Preinstall | reload kubelet
-  when: ansible_os_family != "CoreOS"
+  when: ansible_os_family != "CoreOS" or kube_network_plugin not in ['canal', 'calico']
 
 - name: Preinstall | update resolvconf for CoreOS
   command: /bin/true
@@ -30,10 +24,6 @@
     - Preinstall | reload kubelet
   when: ansible_os_family == "CoreOS"
 
-- name: Preinstall | reload resolvconf
-  command: /sbin/resolvconf -u
-  ignore_errors: true
-
 - name: Preinstall | apply resolvconf cloud-init
   command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
   when: ansible_os_family == "CoreOS"

roles/kubernetes/preinstall/tasks/dhclient-hooks.yml

@@ -0,0 +1,33 @@
+---
+- name: Configure dhclient to prepend nameservers and supersede search/domain
+  blockinfile:
+    block: |-
+      {% for item in [ supersede_domain, supersede_search, prepend_nameserver ] -%}
+      {{ item }}
+      {% endfor %}
+    dest: "{{dhclientconffile}}"
+    create: yes
+    state: present
+    insertbefore: BOF
+    backup: yes
+    follow: yes
+    marker: "# Ansible entries {mark}"
+  notify: Preinstall | restart network
+
+- name: Configue dhclient hooks for resolv.conf (non-RH)
+  template:
+    src: dhclient_dnsupdate.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: 0755
+  notify: Preinstall | restart network
+  when: ansible_os_family != "RedHat"
+
+- name: Configue dhclient hooks for resolv.conf (RH-only)
+  template:
+    src: dhclient_dnsupdate_rh.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: 0755
+  notify: Preinstall | restart network
+  when: ansible_os_family == "RedHat"

roles/kubernetes/preinstall/tasks/resolvconf.yml

@@ -1,120 +1,34 @@
 ---
-- name: check resolvconf
-  shell: which resolvconf
-  register: resolvconf
-  ignore_errors: yes
-  changed_when: false
-  tags: facts
-
-- name: check kubelet
-  stat:
-    path: "{{ bin_dir }}/kubelet"
-  register: kubelet
-  changed_when: false
-  tags: facts
-
-- name: check if early DNS configuration stage
-  set_fact:
-    dns_early: >-
-      {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
-  tags: facts
-
-- name: target resolv.conf file
-  set_fact:
-    resolvconffile: >-
-      {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
-  when: ansible_os_family != "CoreOS"
-  tags: facts
-
-- name: target temporary resolvconf cloud init file
-  set_fact:
-    resolvconffile: /tmp/resolveconf_cloud_init_conf
-  when: ansible_os_family == "CoreOS"
-  tags: facts
-
 - name: create temporary resolveconf cloud init file
   command: cp -f /etc/resolv.conf "{{ resolvconffile }}"
   when: ansible_os_family == "CoreOS"
 
-- name: generate search domains to resolvconf
-  set_fact:
-    searchentries:
-      "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
-  tags: facts
-
-- name: decide on dns server IP
-  set_fact:
-    dns_server_real: >-
-      {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
-
-- name: pick dnsmasq cluster IP or default resolver
-  set_fact:
-    dnsmasq_server: |-
-      {%- if skip_dnsmasq|bool and not dns_early|bool -%}
-        {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
-      {%- elif dns_early|bool -%}
-        {{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
-      {%- else -%}
-        {{ [ dns_server ] }}
-      {%- endif -%}
-  tags: facts
-
-- name: generate nameservers to resolvconf
-  set_fact:
-    nameserverentries:
-      "{{ dnsmasq_server|default([]) + nameservers|default([]) }}"
-  tags: facts
-
-- name: Remove search and nameserver options from resolvconf head
+- name: Remove search/domain/nameserver options
   lineinfile:
-    dest: /etc/resolvconf/resolv.conf.d/head
+    dest: "{{item[0]}}"
     state: absent
-    regexp: "^{{ item }}.*$"
+    regexp: "^{{ item[1] }}.*$"
     backup: yes
     follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: resolvconf.rc == 0
-  notify: Preinstall | update resolvconf
+  with_nested:
+    - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}"
+    - [ 'search ', 'nameserver ', 'domain ', 'options ' ]
+  notify: Preinstall | restart network
 
-- name: Remove search and nameserver options from resolvconf cloud init temporary file
-  lineinfile:
-    dest: "{{resolvconffile}}"
-    state: absent
-    regexp: "^{{ item }}.*$"
-    backup: yes
-    follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: ansible_os_family == "CoreOS"
-  notify: Preinstall | update resolvconf for CoreOS
-
-- name: Add search domains to resolvconf file
-  lineinfile:
-    line: "search {{searchentries}}"
-    dest: "{{resolvconffile}}"
-    state: present
-    insertbefore: BOF
-    backup: yes
-    follow: yes
-  notify: Preinstall | update resolvconf
-
-- name: Add nameservers to resolv.conf
+- name: Add domain/search/nameservers to resolv.conf
   blockinfile:
     dest: "{{resolvconffile}}"
     block: |-
-      {% for item in nameserverentries -%}
-      nameserver {{ item }}
+      {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%}
+      {{ item }}
       {% endfor %}
     state: present
-    insertafter: "^search default.svc.*$"
+    insertbefore: BOF
     create: yes
     backup: yes
     follow: yes
-    marker: "# Ansible nameservers {mark}"
-  notify: Preinstall | update resolvconf
+    marker: "# Ansible entries {mark}"
+  notify: Preinstall | restart network
 
 - name: Add options to resolv.conf
   lineinfile:
@@ -129,30 +43,7 @@
     - ndots:{{ ndots }}
     - timeout:2
     - attempts:2
-  notify: Preinstall | update resolvconf
-
-- name: Remove search and nameserver options from resolvconf base
-  lineinfile:
-    dest: /etc/resolvconf/resolv.conf.d/base
-    state: absent
-    regexp: "^{{ item }}.*$"
-    backup: yes
-    follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: resolvconf.rc == 0
-  notify: Preinstall | update resolvconf
-
-- name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
   notify: Preinstall | restart network
-  when: ansible_os_family == "Debian"
-
-- name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
-  notify: Preinstall | restart network
-  when: ansible_os_family == "RedHat"
 
 - name: get temporary resolveconf cloud init file content
   command: cat {{ resolvconffile }}
@@ -167,3 +58,7 @@
     mode: 0644
   notify: Preinstall | update resolvconf for CoreOS
   when: ansible_os_family == "CoreOS"
+
+- include: dhclient-hooks.yml
+  when: ansible_os_family != "CoreOS"
+  tags: [bootstrap-os, resolvconf]

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -49,6 +49,6 @@
     etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=")
 - set_fact:
     etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}"
-- set_fact:
-    default_resolver: >-
-      {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}
+
+- include: set_resolv_facts.yml
+  tags: [bootstrap-os, resolvconf, facts]

roles/kubernetes/preinstall/tasks/set_resolv_facts.yml

@@ -0,0 +1,88 @@
+---
+- name: check resolvconf
+  shell: which resolvconf
+  register: resolvconf
+  ignore_errors: yes
+  changed_when: false
+
+- set_fact:
+    resolvconf: >-
+      {%- if resolvconf.rc == 0 -%}true{%- else -%}false{%- endif -%}
+
+- set_fact:
+    private_domains: |-
+      {% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%}
+      {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./
+      {%- endfor %}
+    default_resolver: >-
+      {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}
+
+- name: check kubelet
+  stat:
+    path: "{{ bin_dir }}/kubelet"
+  register: kubelet
+  changed_when: false
+
+- name: check if early DNS configuration stage
+  set_fact:
+    dns_early: >-
+      {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
+
+- name: target resolv.conf files
+  set_fact:
+    resolvconffile: /etc/resolv.conf
+    base: >-
+      {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}
+    head: >-
+      {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}
+  when: ansible_os_family != "CoreOS"
+
+- name: target temporary resolvconf cloud init file (CoreOS)
+  set_fact: resolvconffile=/tmp/resolveconf_cloud_init_conf
+  when: ansible_os_family == "CoreOS"
+
+- name: target dhclient conf/hook files for Red Hat family
+  set_fact:
+    dhclientconffile: /etc/dhclient.conf
+    dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh
+  when: ansible_os_family == "RedHat"
+
+- name: target dhclient conf/hook files for Debian family
+  set_fact:
+    dhclientconffile: /etc/dhcp/dhclient.conf
+    dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
+  when: ansible_os_family == "Debian"
+
+- name: generate search domains to resolvconf
+  set_fact:
+    searchentries:
+      search {{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}
+    domainentry:
+      domain {{ dns_domain }}
+    supersede_search:
+      supersede domain-search "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join('", "') }}";
+    supersede_domain:
+      supersede domain-name "{{ dns_domain }}";
+
+- name: decide on dns server IP
+  set_fact:
+    dns_server_real: >-
+      {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
+
+- name: pick dnsmasq cluster IP or default resolver
+  set_fact:
+    dnsmasq_server: |-
+      {%- if skip_dnsmasq|bool and not dns_early|bool -%}
+        {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
+      {%- elif dns_early|bool -%}
+        {{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
+      {%- else -%}
+        {{ [ dns_server ] }}
+      {%- endif -%}
+
+- name: generate nameservers to resolvconf
+  set_fact:
+    nameserverentries:
+      nameserver {{( dnsmasq_server|default([]) + nameservers|default([])) | join(',nameserver ')}}
+    prepend_nameserver:
+      prepend domain-name-servers {{( dnsmasq_server|default([]) + nameservers|default([])) | join(', ') }};

roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2

@@ -0,0 +1,13 @@
+#!/bin/sh
+#
+# Prepend resolver options to /etc/resolv.conf after dhclient`
+# regenerates the file. See man (5) resolver for more details.
+#
+if [ $reason = "BOUND" ]; then
+  if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
+    RESOLV_CONF=$(cat /etc/resolv.conf)
+    OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
+
+    printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
+  fi
+fi

roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2

@@ -0,0 +1,17 @@
+#!/bin/sh
+#
+# Prepend resolver options to /etc/resolv.conf after dhclient`
+# regenerates the file. See man (5) resolver for more details.
+#
+zdnsupdate_config() {
+  if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
+    RESOLV_CONF=$(cat /etc/resolv.conf)
+    OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
+
+    echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
+  fi
+}
+
+zdnsupdate_restore() {
+  :
+}