From 3117858dcdf4888ce13f902edb4d7d647978516a Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobrelia@mirantis.com>
Date: Wed, 30 Nov 2016 14:06:11 +0100
Subject: [PATCH 1/2] Rework DNS stack to meet hostnet pods needs

* For Debian/RedHat OS families (with NetworkManager/dhclient/resolvconf
  optionally enabled) prepend /etc/resolv.conf with required nameservers,
  options, and supersede domain and search domains via the dhclient/resolvconf
  hooks.

* Drop (z)nodnsupdate dhclient hook and re-implement it to complement the
  resolvconf -u command, which is distro/cloud provider specific.
  Update docs as well.

* Enable network restart to apply and persist changes and simplify handlers
  to rely on network restart only. This fixes DNS resolve for hostnet K8s
  pods for Red Hat OS family. Skip network restart for canal/calico plugins,
  unless https://github.com/projectcalico/felix/issues/1185 fixed.

* Replace linefiles line plus with_items to block mode as it's faster.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
Co-authored-by: Matthew Mosesohn <mmosesohn@mirantis.com>
---
 docs/dns-stack.md                             |  11 ++
 roles/kubernetes/preinstall/handlers/main.yml |  16 +-
 .../preinstall/tasks/dhclient-hooks.yml       |  33 +++++
 .../preinstall/tasks/resolvconf.yml           | 139 +++---------------
 .../kubernetes/preinstall/tasks/set_facts.yml |   6 +-
 .../preinstall/tasks/set_resolv_facts.yml     |  88 +++++++++++
 .../templates/dhclient_dnsupdate.sh.j2        |  13 ++
 .../templates/dhclient_dnsupdate_rh.sh.j2     |  17 +++
 8 files changed, 185 insertions(+), 138 deletions(-)
 create mode 100644 roles/kubernetes/preinstall/tasks/dhclient-hooks.yml
 create mode 100644 roles/kubernetes/preinstall/tasks/set_resolv_facts.yml
 create mode 100644 roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2
 create mode 100644 roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2

diff --git a/docs/dns-stack.md b/docs/dns-stack.md
index 9905eb496..54a0081c1 100644
--- a/docs/dns-stack.md
+++ b/docs/dns-stack.md
@@ -38,6 +38,17 @@ or `8.8.8.8`. And domain is set to the default ``dns_domain`` value as well.
 Later, the nameservers will be reconfigured to the DNS service IP that Kargo
 configures for K8s cluster.
 
+Also note, existing records will be purged from the `/etc/resolv.conf`,
+including base/head/cloud-init config files and those that come from dhclient.
+This is required for hostnet pods networking and for [kubelet to not exceed search domains
+limits](https://github.com/kubernetes/kubernetes/issues/9229).
+
+New search, nameserver records and options will be defined from the aforementioned vars:
+* Via resolvconf's head file, if resolvconf installed.
+* Via dhclient's DNS update hook.
+* Via cloud-init (CoreOS only).
+* Statically in the `/etc/resolv.conf`, if none of above is applicable.
+
 DNS configuration details
 -------------------------
 
diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml
index e760a8d8b..481bb1a34 100644
--- a/roles/kubernetes/preinstall/handlers/main.yml
+++ b/roles/kubernetes/preinstall/handlers/main.yml
@@ -2,9 +2,10 @@
   command: /bin/true
   notify:
     - Preinstall | reload network
-    - Preinstall | update resolvconf
+    - Preinstall | reload kubelet
   when: ansible_os_family != "CoreOS"
 
+  # FIXME(bogdando) https://github.com/projectcalico/felix/issues/1185
 - name: Preinstall | reload network
   service:
     name: >-
@@ -14,14 +15,7 @@
       networking
       {%- endif %}
     state: restarted
-  when: ansible_os_family != "RedHat" and ansible_os_family != "CoreOS"
-
-- name: Preinstall | update resolvconf
-  command: /bin/true
-  notify:
-    - Preinstall | reload resolvconf
-    - Preinstall | reload kubelet
-  when: ansible_os_family != "CoreOS"
+  when: ansible_os_family != "CoreOS" or kube_network_plugin not in ['canal', 'calico']
 
 - name: Preinstall | update resolvconf for CoreOS
   command: /bin/true
@@ -30,10 +24,6 @@
     - Preinstall | reload kubelet
   when: ansible_os_family == "CoreOS"
 
-- name: Preinstall | reload resolvconf
-  command: /sbin/resolvconf -u
-  ignore_errors: true
-
 - name: Preinstall | apply resolvconf cloud-init
   command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
   when: ansible_os_family == "CoreOS"
diff --git a/roles/kubernetes/preinstall/tasks/dhclient-hooks.yml b/roles/kubernetes/preinstall/tasks/dhclient-hooks.yml
new file mode 100644
index 000000000..914ffca92
--- /dev/null
+++ b/roles/kubernetes/preinstall/tasks/dhclient-hooks.yml
@@ -0,0 +1,33 @@
+---
+- name: Configure dhclient to prepend nameservers and supersede search/domain
+  blockinfile:
+    block: |-
+      {% for item in [ supersede_domain, supersede_search, prepend_nameserver ] -%}
+      {{ item }}
+      {% endfor %}
+    dest: "{{dhclientconffile}}"
+    create: yes
+    state: present
+    insertbefore: BOF
+    backup: yes
+    follow: yes
+    marker: "# Ansible entries {mark}"
+  notify: Preinstall | restart network
+
+- name: Configue dhclient hooks for resolv.conf (non-RH)
+  template:
+    src: dhclient_dnsupdate.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: 0755
+  notify: Preinstall | restart network
+  when: ansible_os_family != "RedHat"
+
+- name: Configue dhclient hooks for resolv.conf (RH-only)
+  template:
+    src: dhclient_dnsupdate_rh.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: 0755
+  notify: Preinstall | restart network
+  when: ansible_os_family == "RedHat"
diff --git a/roles/kubernetes/preinstall/tasks/resolvconf.yml b/roles/kubernetes/preinstall/tasks/resolvconf.yml
index f82e9ddfb..faf3fc671 100644
--- a/roles/kubernetes/preinstall/tasks/resolvconf.yml
+++ b/roles/kubernetes/preinstall/tasks/resolvconf.yml
@@ -1,120 +1,34 @@
 ---
-- name: check resolvconf
-  shell: which resolvconf
-  register: resolvconf
-  ignore_errors: yes
-  changed_when: false
-  tags: facts
-
-- name: check kubelet
-  stat:
-    path: "{{ bin_dir }}/kubelet"
-  register: kubelet
-  changed_when: false
-  tags: facts
-
-- name: check if early DNS configuration stage
-  set_fact:
-    dns_early: >-
-      {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
-  tags: facts
-
-- name: target resolv.conf file
-  set_fact:
-    resolvconffile: >-
-      {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
-  when: ansible_os_family != "CoreOS"
-  tags: facts
-
-- name: target temporary resolvconf cloud init file
-  set_fact:
-    resolvconffile: /tmp/resolveconf_cloud_init_conf
-  when: ansible_os_family == "CoreOS"
-  tags: facts
-
 - name: create temporary resolveconf cloud init file
   command: cp -f /etc/resolv.conf "{{ resolvconffile }}"
   when: ansible_os_family == "CoreOS"
 
-- name: generate search domains to resolvconf
-  set_fact:
-    searchentries:
-      "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
-  tags: facts
-
-- name: decide on dns server IP
-  set_fact:
-    dns_server_real: >-
-      {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
-
-- name: pick dnsmasq cluster IP or default resolver
-  set_fact:
-    dnsmasq_server: |-
-      {%- if skip_dnsmasq|bool and not dns_early|bool -%}
-        {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
-      {%- elif dns_early|bool -%}
-        {{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
-      {%- else -%}
-        {{ [ dns_server ] }}
-      {%- endif -%}
-  tags: facts
-
-- name: generate nameservers to resolvconf
-  set_fact:
-    nameserverentries:
-      "{{ dnsmasq_server|default([]) + nameservers|default([]) }}"
-  tags: facts
-
-- name: Remove search and nameserver options from resolvconf head
+- name: Remove search/domain/nameserver options
   lineinfile:
-    dest: /etc/resolvconf/resolv.conf.d/head
+    dest: "{{item[0]}}"
     state: absent
-    regexp: "^{{ item }}.*$"
+    regexp: "^{{ item[1] }}.*$"
     backup: yes
     follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: resolvconf.rc == 0
-  notify: Preinstall | update resolvconf
+  with_nested:
+    - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}"
+    - [ 'search ', 'nameserver ', 'domain ', 'options ' ]
+  notify: Preinstall | restart network
 
-- name: Remove search and nameserver options from resolvconf cloud init temporary file
-  lineinfile:
-    dest: "{{resolvconffile}}"
-    state: absent
-    regexp: "^{{ item }}.*$"
-    backup: yes
-    follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: ansible_os_family == "CoreOS"
-  notify: Preinstall | update resolvconf for CoreOS
-
-- name: Add search domains to resolvconf file
-  lineinfile:
-    line: "search {{searchentries}}"
-    dest: "{{resolvconffile}}"
-    state: present
-    insertbefore: BOF
-    backup: yes
-    follow: yes
-  notify: Preinstall | update resolvconf
-
-- name: Add nameservers to resolv.conf
+- name: Add domain/search/nameservers to resolv.conf
   blockinfile:
     dest: "{{resolvconffile}}"
     block: |-
-      {% for item in nameserverentries -%}
-      nameserver {{ item }}
+      {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%}
+      {{ item }}
       {% endfor %}
     state: present
-    insertafter: "^search default.svc.*$"
+    insertbefore: BOF
     create: yes
     backup: yes
     follow: yes
-    marker: "# Ansible nameservers {mark}"
-  notify: Preinstall | update resolvconf
+    marker: "# Ansible entries {mark}"
+  notify: Preinstall | restart network
 
 - name: Add options to resolv.conf
   lineinfile:
@@ -129,30 +43,7 @@
     - ndots:{{ ndots }}
     - timeout:2
     - attempts:2
-  notify: Preinstall | update resolvconf
-
-- name: Remove search and nameserver options from resolvconf base
-  lineinfile:
-    dest: /etc/resolvconf/resolv.conf.d/base
-    state: absent
-    regexp: "^{{ item }}.*$"
-    backup: yes
-    follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: resolvconf.rc == 0
-  notify: Preinstall | update resolvconf
-
-- name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
   notify: Preinstall | restart network
-  when: ansible_os_family == "Debian"
-
-- name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
-  notify: Preinstall | restart network
-  when: ansible_os_family == "RedHat"
 
 - name: get temporary resolveconf cloud init file content
   command: cat {{ resolvconffile }}
@@ -167,3 +58,7 @@
     mode: 0644
   notify: Preinstall | update resolvconf for CoreOS
   when: ansible_os_family == "CoreOS"
+
+- include: dhclient-hooks.yml
+  when: ansible_os_family != "CoreOS"
+  tags: [bootstrap-os, resolvconf]
diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml
index 0f66acd14..cbe4d9203 100644
--- a/roles/kubernetes/preinstall/tasks/set_facts.yml
+++ b/roles/kubernetes/preinstall/tasks/set_facts.yml
@@ -49,6 +49,6 @@
     etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=")
 - set_fact:
     etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}"
-- set_fact:
-    default_resolver: >-
-      {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}
+
+- include: set_resolv_facts.yml
+  tags: [bootstrap-os, resolvconf, facts]
diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml
new file mode 100644
index 000000000..288c80afe
--- /dev/null
+++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml
@@ -0,0 +1,88 @@
+---
+- name: check resolvconf
+  shell: which resolvconf
+  register: resolvconf
+  ignore_errors: yes
+  changed_when: false
+
+- set_fact:
+    resolvconf: >-
+      {%- if resolvconf.rc == 0 -%}true{%- else -%}false{%- endif -%}
+
+- set_fact:
+    private_domains: |-
+      {% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%}
+      {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./
+      {%- endfor %}
+    default_resolver: >-
+      {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}
+
+- name: check kubelet
+  stat:
+    path: "{{ bin_dir }}/kubelet"
+  register: kubelet
+  changed_when: false
+
+- name: check if early DNS configuration stage
+  set_fact:
+    dns_early: >-
+      {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
+
+- name: target resolv.conf files
+  set_fact:
+    resolvconffile: /etc/resolv.conf
+    base: >-
+      {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}
+    head: >-
+      {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}
+  when: ansible_os_family != "CoreOS"
+
+- name: target temporary resolvconf cloud init file (CoreOS)
+  set_fact: resolvconffile=/tmp/resolveconf_cloud_init_conf
+  when: ansible_os_family == "CoreOS"
+
+- name: target dhclient conf/hook files for Red Hat family
+  set_fact:
+    dhclientconffile: /etc/dhclient.conf
+    dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh
+  when: ansible_os_family == "RedHat"
+
+- name: target dhclient conf/hook files for Debian family
+  set_fact:
+    dhclientconffile: /etc/dhcp/dhclient.conf
+    dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
+  when: ansible_os_family == "Debian"
+
+- name: generate search domains to resolvconf
+  set_fact:
+    searchentries:
+      search {{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}
+    domainentry:
+      domain {{ dns_domain }}
+    supersede_search:
+      supersede domain-search "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join('", "') }}";
+    supersede_domain:
+      supersede domain-name "{{ dns_domain }}";
+
+- name: decide on dns server IP
+  set_fact:
+    dns_server_real: >-
+      {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
+
+- name: pick dnsmasq cluster IP or default resolver
+  set_fact:
+    dnsmasq_server: |-
+      {%- if skip_dnsmasq|bool and not dns_early|bool -%}
+        {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
+      {%- elif dns_early|bool -%}
+        {{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
+      {%- else -%}
+        {{ [ dns_server ] }}
+      {%- endif -%}
+
+- name: generate nameservers to resolvconf
+  set_fact:
+    nameserverentries:
+      nameserver {{( dnsmasq_server|default([]) + nameservers|default([])) | join(',nameserver ')}}
+    prepend_nameserver:
+      prepend domain-name-servers {{( dnsmasq_server|default([]) + nameservers|default([])) | join(', ') }};
diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2
new file mode 100644
index 000000000..84eb23926
--- /dev/null
+++ b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2
@@ -0,0 +1,13 @@
+#!/bin/sh
+#
+# Prepend resolver options to /etc/resolv.conf after dhclient`
+# regenerates the file. See man (5) resolver for more details.
+#
+if [ $reason = "BOUND" ]; then
+  if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
+    RESOLV_CONF=$(cat /etc/resolv.conf)
+    OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
+
+    printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
+  fi
+fi
diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2
new file mode 100644
index 000000000..514863663
--- /dev/null
+++ b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2
@@ -0,0 +1,17 @@
+#!/bin/sh
+#
+# Prepend resolver options to /etc/resolv.conf after dhclient`
+# regenerates the file. See man (5) resolver for more details.
+#
+zdnsupdate_config() {
+  if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
+    RESOLV_CONF=$(cat /etc/resolv.conf)
+    OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
+
+    echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
+  fi
+}
+
+zdnsupdate_restore() {
+  :
+}

From f52ed9f91ea922fd239c00168d83f19d3bdc952f Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bogdando@mail.ru>
Date: Mon, 12 Dec 2016 21:37:16 +0100
Subject: [PATCH 2/2] Update main.yml

---
 roles/kubernetes/preinstall/handlers/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml
index 481bb1a34..81d690bb6 100644
--- a/roles/kubernetes/preinstall/handlers/main.yml
+++ b/roles/kubernetes/preinstall/handlers/main.yml
@@ -15,7 +15,7 @@
       networking
       {%- endif %}
     state: restarted
-  when: ansible_os_family != "CoreOS" or kube_network_plugin not in ['canal', 'calico']
+  when: ansible_os_family != "CoreOS" and kube_network_plugin not in ['canal', 'calico']
 
 - name: Preinstall | update resolvconf for CoreOS
   command: /bin/true