containerd: change default resolvconf_mode to host_resolvconf (#8247)

* containerd: change default resolvconf_mode to host_resolvconf

* Wait for kube-apiserver to come back after pod refresh

* Handle resolv.conf gracefully

* Retain currently configured DNS entries to ensure we don't break the resolvers

* Suse uses wickedd for network management so no dhcp hooks

* Molecule: increase ansible timeout

* CI: Increase ansible timeout to 120s for Packet jobs

.gitlab-ci/packet.yml

@@ -2,6 +2,7 @@
 .packet:
   extends: .testcases
   variables:
+    ANSIBLE_TIMEOUT: "120"
     CI_PLATFORM: packet
     SSH_USER: kubespray
   tags:

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -192,7 +192,7 @@ coredns_k8s_external_zone: k8s_external.local
 enable_coredns_k8s_endpoint_pod_names: false
 
 # Can be docker_dns, host_resolvconf or none
-resolvconf_mode: docker_dns
+resolvconf_mode: host_resolvconf
 # Deploy netchecker app to verify DNS resolve as an HTTP service
 deploy_netchecker: false
 # Ip address of the kubernetes skydns service

roles/adduser/molecule/default/molecule.yml

@@ -15,6 +15,10 @@ platforms:
     memory: 512
 provisioner:
   name: ansible
+  config_options:
+    defaults:
+      callback_whitelist: profile_tasks
+      timeout: 120
   lint:
     name: ansible-lint
 verifier:

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -15,6 +15,10 @@ platforms:
     memory: 512
 provisioner:
   name: ansible
+  config_options:
+    defaults:
+      callback_whitelist: profile_tasks
+      timeout: 120
   lint:
     name: ansible-lint
   inventory:

roles/bootstrap-os/molecule/default/molecule.yml

@@ -35,6 +35,10 @@ platforms:
     memory: 512
 provisioner:
   name: ansible
+  config_options:
+    defaults:
+      callback_whitelist: profile_tasks
+      timeout: 120
   lint:
     name: ansible-lint
   inventory:

roles/container-engine/containerd/molecule/default/molecule.yml

@@ -46,6 +46,7 @@ provisioner:
   config_options:
     defaults:
       callback_whitelist: profile_tasks
+      timeout: 120
   lint:
     name: ansible-lint
     options:

roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -38,6 +38,7 @@ provisioner:
   config_options:
     defaults:
       callback_whitelist: profile_tasks
+      timeout: 120
   lint:
     name: ansible-lint
     options:

roles/container-engine/docker/molecule/default/molecule.yml

@@ -18,6 +18,7 @@ provisioner:
   config_options:
     defaults:
       callback_whitelist: profile_tasks
+      timeout: 120
   lint:
     name: ansible-lint
     options:

roles/container-engine/gvisor/molecule/default/molecule.yml

@@ -30,6 +30,7 @@ provisioner:
   config_options:
     defaults:
       callback_whitelist: profile_tasks
+      timeout: 120
   lint:
     name: ansible-lint
     options:

roles/container-engine/kata-containers/molecule/default/molecule.yml

@@ -30,6 +30,7 @@ provisioner:
   config_options:
     defaults:
       callback_whitelist: profile_tasks
+      timeout: 120
   lint:
     name: ansible-lint
     options:

roles/kubernetes/preinstall/handlers/main.yml

@@ -9,6 +9,7 @@
     - Preinstall | restart kube-controller-manager crio/containerd
     - Preinstall | restart kube-apiserver docker
     - Preinstall | restart kube-apiserver crio/containerd
+    - Preinstall | wait for the apiserver to be running
   when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos
 
 - name: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk
@@ -101,6 +102,21 @@
     - dns_mode != 'none'
     - resolvconf_mode == 'host_resolvconf'
 
+# When running this as the last phase ensure we wait for kube-apiserver to come up
+- name: Preinstall | wait for the apiserver to be running
+  uri:
+    url: "{{ kube_apiserver_endpoint }}/healthz"
+    validate_certs: no
+  register: result
+  until: result.status == 200
+  retries: 60
+  delay: 1
+  when:
+    - dns_late
+    - inventory_hostname in groups['kube_control_plane']
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+
 - name: Preinstall | Restart systemd-resolved
   service:
     name: systemd-resolved

roles/kubernetes/preinstall/tasks/0040-set_facts.yml

@@ -34,6 +34,39 @@
   changed_when: false
   check_mode: no
 
+- name: check existence of /etc/resolvconf/resolv.conf.d
+  stat:
+    path: /etc/resolvconf/resolv.conf.d
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  failed_when: false
+  register: resolvconfd_path
+
+- name: check status of /etc/resolv.conf
+  stat:
+    path: /etc/resolv.conf
+    follow: no
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  failed_when: false
+  register: resolvconf_stat
+
+- block:
+
+    - name: get content of /etc/resolv.conf
+      slurp:
+        src: /etc/resolv.conf
+      register: resolvconf_slurp
+
+    - name: get currently configured nameservers
+      set_fact:
+        configured_nameservers: "{{ resolvconf_slurp.content | b64decode | regex_findall('\\s*nameserver\\s*(.*)') | ipaddr }}"
+      when: resolvconf_slurp.content is defined
+
+  when: resolvconf_stat.stat.exists is defined and resolvconf_stat.stat.exists
+
 - name: check systemd-resolved
   # noqa 303 Should we use service_facts for this?
   command: systemctl is-active systemd-resolved
@@ -45,7 +78,7 @@
 - name: set dns facts
   set_fact:
     resolvconf: >-
-      {%- if resolvconf.rc == 0 -%}true{%- else -%}false{%- endif -%}
+      {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%}
     bogus_domains: |-
       {% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%}
       {{ dns_domain }}.{{ d }}./{{ d }}.{{ d }}./com.{{ d }}./
@@ -147,7 +180,7 @@
 - name: generate nameservers to resolvconf
   set_fact:
     nameserverentries:
-      nameserver {{ ( ( [nodelocaldns_ip] if enable_nodelocaldns else []) + coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(',nameserver ') }}
+      nameserver {{ ( ( [nodelocaldns_ip] if enable_nodelocaldns else []) + coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',nameserver ') }}
     supersede_nameserver:
       supersede domain-name-servers {{ ( coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(', ') }};
 

roles/kubernetes/preinstall/tasks/0060-resolvconf.yml

@@ -16,7 +16,7 @@
     state: present
     insertbefore: BOF
     create: yes
-    backup: yes
+    backup: "{{ not resolvconf_stat.stat.islnk }}"
     marker: "# Ansible entries {mark}"
     mode: 0644
   notify: Preinstall | propagate resolvconf to k8s components
@@ -25,7 +25,7 @@
   replace:
     path: "{{ item[0] }}"
     regexp: '^{{ item[1] }}[^#]*(?=# Ansible entries BEGIN)'
-    backup: yes
+    backup: "{{ not resolvconf_stat.stat.islnk }}"
   with_nested:
     - "{{ [resolvconffile, base|default(''), head|default('')] | difference(['']) }}"
     - [ 'search ', 'nameserver ', 'domain ', 'options ' ]
@@ -36,13 +36,12 @@
     path: "{{ item[0] }}"
     regexp: '(# Ansible entries END\n(?:(?!^{{ item[1] }}).*\n)*)(?:^{{ item[1] }}.*\n?)+'
     replace: '\1'
-    backup: yes
+    backup: "{{ not resolvconf_stat.stat.islnk }}"
   with_nested:
     - "{{ [resolvconffile, base|default(''), head|default('')] | difference(['']) }}"
     - [ 'search ', 'nameserver ', 'domain ', 'options ' ]
   notify: Preinstall | propagate resolvconf to k8s components
 
-
 - name: get temporary resolveconf cloud init file content
   command: cat {{ resolvconffile }}
   register: cloud_config

roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml

@@ -22,7 +22,7 @@
     owner: root
     mode: 0755
   notify: Preinstall | propagate resolvconf to k8s components
-  when: ansible_os_family != "RedHat"
+  when: ansible_os_family not in [ "RedHat", "Suse" ]
 
 - name: Configure dhclient hooks for resolv.conf (RH-only)
   template:

roles/kubespray-defaults/defaults/main.yaml

@@ -106,7 +106,7 @@ nodelocaldns_secondary_skew_seconds: 5
 manual_dns_server: ""
 
 # Can be docker_dns, host_resolvconf or none
-resolvconf_mode: docker_dns
+resolvconf_mode: host_resolvconf
 # Deploy netchecker app to verify DNS resolve as an HTTP service
 deploy_netchecker: false
 # Ip address of the kubernetes DNS service (called skydns for historical reasons)

tests/files/packet_centos7-docker-weave-upgrade-ha.yml

@@ -10,6 +10,7 @@ kubernetes_audit: true
 # Docker specific settings:
 container_manager: docker
 etcd_deployment_type: docker
+resolvconf_mode: docker_dns
 
 # Needed to upgrade from 1.16 to 1.17, otherwise upgrade is partial and bug followed
 upgrade_cluster_setup: true

tests/files/packet_centos8-docker.yml

@@ -10,3 +10,4 @@ calico_iptables_backend: "Auto"
 # Use docker
 container_manager: docker
 etcd_deployment_type: docker
+resolvconf_mode: docker_dns

tests/files/packet_debian10-docker.yml

@@ -6,3 +6,4 @@ mode: default
 # Use docker
 container_manager: docker
 etcd_deployment_type: docker
+resolvconf_mode: docker_dns

tests/files/packet_debian11-docker.yml

@@ -6,3 +6,4 @@ mode: default
 # Use docker
 container_manager: docker
 etcd_deployment_type: docker
+resolvconf_mode: docker_dns

tests/files/packet_fedora34-docker-weave.yml

@@ -9,3 +9,4 @@ kube_network_plugin: weave
 # Docker specific settings:
 container_manager: docker
 etcd_deployment_type: docker
+resolvconf_mode: docker_dns

tests/files/packet_ubuntu16-docker-weave-sep.yml

@@ -10,6 +10,7 @@ auto_renew_certificates: true
 # Docker specific settings:
 container_manager: docker
 etcd_deployment_type: docker
+resolvconf_mode: docker_dns
 
 # Ubuntu 16 - docker containerd package available stopped at 1.4.6
 docker_containerd_version: latest

tests/files/packet_ubuntu18-docker.yml

@@ -7,3 +7,4 @@ vm_memory: 1600Mi
 # Use docker
 container_manager: docker
 etcd_deployment_type: docker
+resolvconf_mode: docker_dns

tests/files/packet_ubuntu20-docker.yml

@@ -14,3 +14,4 @@ enable_nodelocaldns: False
 # Use docker
 container_manager: docker
 etcd_deployment_type: docker
+resolvconf_mode: docker_dns