Remove hard dependence on facts for all nodes (#4304)

* Remove hard dependence on facts for all nodes

* Update main.yaml

* Update main.yaml

roles/etcd/tasks/join_etcd-events_member.yml

@@ -15,7 +15,7 @@
     etcd_events_peer_addresses: >-
       {% for host in groups['etcd'] -%}
         {%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%}
-          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2382,
+          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(fallback_ips[host])) }}:2382,
         {%- endif -%}
         {%- if loop.last -%}
           {{ etcd_member_name }}={{ etcd_events_peer_url }}

roles/etcd/tasks/join_etcd_member.yml

@@ -16,7 +16,7 @@
     etcd_peer_addresses: >-
       {% for host in groups['etcd'] -%}
         {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}
-          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2380,
+          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(fallback_ips[host])) }}:2380,
         {%- endif -%}
         {%- if loop.last -%}
           {{ etcd_member_name }}={{ etcd_peer_url }}

roles/etcd/templates/openssl.conf.j2

@@ -37,7 +37,7 @@ DNS.{{ counter["dns"] }} = {{ etcd_alt_name }}{{ increment(counter, 'dns') }}
 {% if hostvars[host]['access_ip'] is defined  %}
 IP.{{ counter["ip"] }} = {{ hostvars[host]['access_ip'] }}{{ increment(counter, 'ip') }}
 {% endif %}
-IP.{{ counter["ip"] }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}{{ increment(counter, 'ip') }}
+IP.{{ counter["ip"] }} = {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ increment(counter, 'ip') }}
 {% endfor %}
 {% for cert_alt_ip in etcd_cert_alt_ips %}
 IP.{{ counter["ip"] }} = {{ cert_alt_ip }}{{ increment(counter, 'ip') }}

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -61,7 +61,7 @@
       {%- if hostvars[host]['access_ip'] is defined %}
       {{ hostvars[host]['access_ip'] }}
       {% endif %}
-      {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+      {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}
       {%- endfor %}
       {%- if supplementary_addresses_in_ssl_keys is defined -%}
       {% for addr in supplementary_addresses_in_ssl_keys %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2

@@ -5,7 +5,7 @@ api:
   controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}
   bindPort: {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
 {% else %}
-  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
   bindPort: {{ kube_apiserver_port }}
 {% endif %}
 etcd:

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -5,7 +5,7 @@ api:
   controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}
   bindPort: {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
 {% else %}
-  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
   bindPort: {{ kube_apiserver_port }}
 {% endif %}
 etcd:

roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2

@@ -1,7 +1,7 @@
 apiVersion: kubeadm.k8s.io/v1alpha3
 kind: InitConfiguration
 apiEndpoint:
-  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
   bindPort: {{ kube_apiserver_port }}
 nodeRegistration:
 {% if kube_override_hostname|default('') %}
@@ -40,7 +40,7 @@ kubernetesVersion: {{ kube_version }}
 {% if kubeadm_config_api_fqdn is defined %}
 controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
 {% else %}
-controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }}
+controlPlaneEndpoint: {{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}
 {% endif %}
 apiServerCertSANs:
 {% for san in apiserver_sans.split() | unique %}

roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2

@@ -1,7 +1,7 @@
 apiVersion: kubeadm.k8s.io/v1beta1
 kind: InitConfiguration
 localAPIEndpoint:
-  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
   bindPort: {{ kube_apiserver_port }}
 nodeRegistration:
 {% if kube_override_hostname|default('') %}
@@ -40,7 +40,7 @@ kubernetesVersion: {{ kube_version }}
 {% if kubeadm_config_api_fqdn is defined %}
 controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
 {% else %}
-controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }}
+controlPlaneEndpoint: {{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}
 {% endif %}
 certificatesDir: {{ kube_cert_dir }}
 imageRepository: {{ kube_image_repo }}

roles/kubernetes/node/defaults/main.yml

@@ -3,7 +3,7 @@
 kube_apiserver_insecure_bind_address: 127.0.0.1
 
 # advertised host IP for kubelet. This affects network plugin config. Take caution
-kubelet_address: "{{ ip | default(ansible_default_ipv4['address']) }}"
+kubelet_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}"
 
 # bind address for kubelet. Set to 0.0.0.0 to listen on all interfaces
 kubelet_bind_address: "{{ ip | default('0.0.0.0') }}"

roles/kubernetes/node/templates/nginx.conf.j2

@@ -11,7 +11,7 @@ stream {
         upstream kube_apiserver {
             least_conn;
             {% for host in groups['kube-master'] -%}
-            server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }};
+            server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }};
             {% endfor %}
         }
 

roles/kubernetes/preinstall/tasks/0090-etchosts.yml

@@ -3,7 +3,11 @@
   blockinfile:
     path: /etc/hosts
     block: |-
-      {% for item in (groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%}{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}{% if (item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }}{% endif %} {{ item }} {{ item }}.{{ dns_domain }}
+      {% for item in (groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%}
+      {% if 'access_ip' in hostvars[item] or 'ip' in hostvars[item] or fallback_ips[item] != "skip" -%}
+      {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item]))}}
+      {%- if ('ansible_hostname' in hostvars[item] and item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }}{% endif %} {{ item }} {{ item }}.{{ dns_domain }}
+      {% endif %}
       {% endfor %}
     state: present
     create: yes

roles/kubespray-defaults/defaults/main.yaml

@@ -358,6 +358,14 @@ contiv_peer_with_uplink_leaf: false
 contiv_global_as: "65002"
 contiv_global_neighbor_as: "500"
 
+# Set 127.0.0.1 as fallback IP if we do not have host facts for host
+fallback_ips_base: |
+  ---
+  {% for item in groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([])|unique %}
+  {{item}}: "{{ hostvars[item].get('ansible_default_ipv4', {'address': '127.0.0.1'})['address'] }}"
+  {% endfor %}
+fallback_ips: "{{ fallback_ips_base | from_yaml }}"
+
 ## Set no_proxy to all assigned cluster IPs and hostnames
 no_proxy: >-
   {%- if http_proxy is defined or https_proxy is defined %}
@@ -366,8 +374,8 @@ no_proxy: >-
   {{ loadbalancer_apiserver.address | default('') }},
   {%- endif -%}
   {%- for item in (groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%}
-  {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }},
-  {%-   if (item != hostvars[item]['ansible_hostname']) -%}
+  {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }},
+  {%-   if item != hostvars[item].get('ansible_hostname', "") -%}
   {{ hostvars[item]['ansible_hostname'] }},
   {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }},
   {%-   endif -%}
@@ -399,9 +407,9 @@ ssl_ca_dirs: >-
 # Vars for pointing to kubernetes api endpoints
 is_kube_master: "{{ inventory_hostname in groups['kube-master'] }}"
 kube_apiserver_count: "{{ groups['kube-master'] | length }}"
-kube_apiserver_address: "{{ ip | default(ansible_default_ipv4['address']) }}"
+kube_apiserver_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}"
 kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}"
-first_kube_master: "{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}"
+first_kube_master: "{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(fallback_ips[groups['kube-master'][0]])) }}"
 loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}"
 # applied if only external loadbalancer_apiserver is defined, otherwise ignored
 apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local"
@@ -425,7 +433,7 @@ etcd_events_cluster_enabled: false
 
 # Vars for pointing to etcd endpoints
 is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"
-etcd_address: "{{ ip | default(ansible_default_ipv4['address']) }}"
+etcd_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}"
 etcd_access_address: "{{ access_ip | default(etcd_address) }}"
 etcd_events_access_address: "{{ access_ip | default(etcd_address) }}"
 etcd_peer_url: "https://{{ etcd_access_address }}:2380"
@@ -434,12 +442,12 @@ etcd_events_peer_url: "https://{{ etcd_events_access_address }}:2382"
 etcd_events_client_url: "https://{{ etcd_events_access_address }}:2381"
 etcd_access_addresses: |-
   {% for item in groups['etcd'] -%}
-    https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}
+    https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }}:2379{% if not loop.last %},{% endif %}
   {%- endfor %}
 etcd_events_access_addresses_list: |-
   [
   {% for item in groups['etcd'] -%}
-    'https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2381'{% if not loop.last %},{% endif %}
+    'https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }}:2381'{% if not loop.last %},{% endif %}
   {%- endfor %}
   ]
 etcd_events_access_addresses: "{{etcd_events_access_addresses_list | join(',')}}"
@@ -451,11 +459,11 @@ etcd_member_name: |-
   {% endfor %}
 etcd_peer_addresses: |-
   {% for item in groups['etcd'] -%}
-    {{ hostvars[item].etcd_member_name | default("etcd" + loop.index|string) }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
+    {{ hostvars[item].etcd_member_name | default("etcd" + loop.index|string) }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(fallback_ips[item])) }}:2380{% if not loop.last %},{% endif %}
   {%- endfor %}
 etcd_events_peer_addresses: |-
   {% for item in groups['etcd'] -%}
-    {{ hostvars[item].etcd_member_name | default("etcd" + loop.index|string) }}-events=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2382{% if not loop.last %},{% endif %}
+    {{ hostvars[item].etcd_member_name | default("etcd" + loop.index|string) }}-events=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(fallback_ips[item])) }}:2382{% if not loop.last %},{% endif %}
   {%- endfor %}
 
 podsecuritypolicy_enabled: false

roles/network_plugin/calico/rr/tasks/main.yml

@@ -5,7 +5,7 @@
 
 - name: Calico-rr | Set IP fact
   set_fact:
-    rr_ip: "{{ calico_rr_ip | default(ip) | default(ansible_default_ipv4.address) }}"
+    rr_ip: "{{ calico_rr_ip | default(ip) | default(fallback_ips[inventory_hostname]) }}"
 
 - name: Calico-rr | Create calico certs directory
   file:

roles/network_plugin/calico/tasks/install.yml

@@ -297,12 +297,12 @@
    "apiVersion": "projectcalico.org/v3",
    "kind": "BGPPeer",
    "metadata": {
-      "name": "{{ inventory_hostname }}-{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"
+      "name": "{{ inventory_hostname }}-{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"
    },
    "spec": {
       "asNumber": "{{ local_as | default(global_as_num)}}",
       "node": "{{ inventory_hostname }}",
-      "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"
+      "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"
    }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
@@ -322,7 +322,7 @@
    "apiVersion": "v1",
    "metadata": {"node": "{{ inventory_hostname }}",
      "scope": "node",
-     "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"}
+     "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"}
    }'
    | {{ bin_dir }}/calicoctl create --skip-exists -f -
   retries: 4

roles/network_plugin/contiv/defaults/main.yml

@@ -14,7 +14,7 @@ contiv_etcd_listen_port: 6666
 contiv_etcd_peer_port: 6667
 contiv_etcd_endpoints: |-
   {% for host in groups['kube-master'] -%}
-    contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(hostvars[host].ansible_default_ipv4['address']) }}:{{ contiv_etcd_peer_port }}{% if not loop.last %},{% endif %}
+    contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(fallback_ips[host]) }}:{{ contiv_etcd_peer_port }}{% if not loop.last %},{% endif %}
   {%- endfor %}
 
 # Parameters for Contiv api-proxy

roles/network_plugin/flannel/defaults/main.yml

@@ -2,7 +2,7 @@
 # Flannel public IP
 # The address that flannel should advertise as how to access the system
 # Disabled until https://github.com/coreos/flannel/issues/712 is fixed
-# flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}"
+# flannel_public_ip: "{{ access_ip|default(ip|default(fallback_ips[inventory_hostname])) }}"
 
 ## interface that should be used for flannel operations
 ## This is actually an inventory cluster-level item