From fd9bbcb157b5ac036c4e67360ce0d0bc611f84b6 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <matthew.mosesohn@gmail.com>
Date: Mon, 15 Jul 2019 11:59:06 +0300
Subject: [PATCH] Enable nodes to run calicoctl for calico kdd mode (#4956)

* Enable nodes to run calicoctl

per-node tasks require waiting for calico-node to be applied

Change-Id: Ibe1076b7334a2da0332f2dd766fde0c3f172d1f2

* cleanup tasks that should run on master

Change-Id: I43a837879ef41596f14657ecd7f813899b6865ae

* Switch run_once calico logic to just run on first master

Change-Id: I6893711e354f63c5e1eaf6ac2e23d9a6347a555d
---
 .../network_plugin/calico/tasks/main.yml      |  17 --
 roles/network_plugin/calico/tasks/install.yml | 190 ++++++++++--------
 roles/network_plugin/calico/tasks/main.yml    |   4 +-
 .../calico/templates/calicoctl.kdd.sh.j2      |   4 +-
 4 files changed, 109 insertions(+), 106 deletions(-)

diff --git a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
index 65fb9d515..9528aa02d 100644
--- a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
+++ b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
@@ -1,21 +1,4 @@
 ---
-- name: Start Calico resources
-  kube:
-    name: "{{ item.item.name }}"
-    namespace: "kube-system"
-    kubectl: "{{ bin_dir }}/kubectl"
-    resource: "{{ item.item.type }}"
-    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
-    state: "latest"
-  with_items:
-    - "{{ calico_node_manifests.results }}"
-    - "{{ calico_node_kdd_manifest.results }}"
-    - "{{ calico_node_typha_manifest.results }}"
-  when:
-    - inventory_hostname == groups['kube-master'][0] and not item is skipped
-  loop_control:
-    label: "{{ item.item.file }}"
-
 - name: "calico upgrade complete"
   shell: "{{ bin_dir }}/calico-upgrade complete --no-prompts --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml"
   when:
diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml
index 287552640..543fa5e37 100644
--- a/roles/network_plugin/calico/tasks/install.yml
+++ b/roles/network_plugin/calico/tasks/install.yml
@@ -21,6 +21,7 @@
     mode: 0750
     owner: root
     group: root
+  when: calico_datastore == "etcd"
 
 - name: Calico | Link etcd certificates for calico-node
   file:
@@ -32,6 +33,7 @@
     - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
     - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
     - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
+  when: calico_datastore == "etcd"
 
 - name: Calico | Install calicoctl wrapper script
   template:
@@ -52,6 +54,7 @@
   retries: 10
   delay: 5
   run_once: true
+  when: calico_datastore == "etcd"
 
 - name: Calico | Check if calico network pool has already been configured
   shell: >
@@ -59,17 +62,16 @@
   register: calico_conf
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
-  delegate_to: "{{ groups['kube-master'][0] }}"
-  run_once: true
   changed_when: false
+  when:
+    - inventory_hostname == groups['kube-master'][0]
 
 - name: Calico | Ensure that calico_pool_cidr is within kube_pods_subnet when defined
   assert:
     that: "[calico_pool_cidr] | ipaddr(kube_pods_subnet) | length == 1"
     msg: "{{ calico_pool_cidr }} is not within or equal to {{ kube_pods_subnet }}"
-  delegate_to: localhost
-  run_once: true
   when:
+    - inventory_hostname == groups['kube-master'][0]
     - 'calico_conf.stdout == "0"'
     - calico_pool_cidr is defined
 
@@ -84,7 +86,7 @@
     - inventory_hostname in groups['kube-master']
     - calico_datastore == "kdd"
 
-- name: Start Calico resources
+- name: Calico | Create Calico Kubernetes datastore resources
   kube:
     name: "{{ item.item.name }}"
     namespace: "kube-system"
@@ -95,7 +97,8 @@
   with_items:
     - "{{ calico_node_kdd_manifest.results }}"
   when:
-    - inventory_hostname == groups['kube-master'][0] and not item is skipped
+    - inventory_hostname == groups['kube-master'][0]
+    - not item is skipped
   loop_control:
     label: "{{ item.item.file }}"
 
@@ -111,9 +114,8 @@
           "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}",
           "ipipMode": "{{ ipip_mode }}",
           "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh create -f -
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
   when:
+    - inventory_hostname == groups['kube-master'][0]
     - 'calico_conf.stdout == "0"'
     - calico_version is version("v3.0.0", ">=")
     - calico_version is version("v3.3.0", "<")
@@ -131,9 +133,8 @@
           "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}",
           "ipipMode": "{{ ipip_mode }}",
           "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh create -f -
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
   when:
+    - inventory_hostname == groups['kube-master'][0]
     - 'calico_conf.stdout == "0"'
     - calico_version is version("v3.3.0", ">=")
 
@@ -148,9 +149,8 @@
       }' | {{ bin_dir }}/calicoctl.sh apply -f -
   environment:
     NO_DEFAULT_POOLS: true
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
   when:
+    - inventory_hostname == groups['kube-master'][0]
     - 'calico_conf.stdout == "0"'
     - calico_version is version("v3.0.0", "<")
 
@@ -174,25 +174,113 @@
         "logSeverityScreen": "Info",
         "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} ,
         "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
   changed_when: false
   when:
+    - inventory_hostname == groups['kube-master'][0]
     - calico_version is version('v3.0.0', '>=')
 
 - name: Calico | Set global as_num (legacy)
   command: "{{ bin_dir }}/calicoctl.sh config set asNumber {{ global_as_num }}"
-  run_once: true
   when:
+    - inventory_hostname == groups['kube-master'][0]
     - calico_version is version('v3.0.0', '<')
 
 - name: Calico | Disable node mesh (legacy)
   command: "{{ bin_dir }}/calicoctl.sh config set nodeToNodeMesh off"
-  run_once: yes
   when:
+    - inventory_hostname == groups['kube-master'][0]
     - calico_version is version('v3.0.0', '<')
     - nodeToMeshEnabled|default(True)
 
+- name: Calico | Configure peering with router(s) at global scope
+  shell: >
+   echo '{
+   "apiVersion": "projectcalico.org/v3",
+   "kind": "BGPPeer",
+   "metadata": {
+      "name": "global-{{ item.router_id }}"
+   },
+   "spec": {
+      "asNumber": "{{ item.as }}",
+      "peerIP": "{{ item.router_id }}"
+   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|list|default([]) }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - calico_version | version_compare('v3.0.0', '>=')
+    - peer_with_router|default(false)
+
+- name: Calico | Configure peering with router(s) at global scope (legacy)
+  shell: >
+   echo '{
+   "kind": "bgpPeer",
+   "spec": {"asNumber": "{{ item.as }}"},
+   "apiVersion": "v1",
+   "metadata": {"scope": "global", "peerIP": "{{ item.router_id }}"}
+   }'
+   | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items: "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|default([]) }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - calico_version is version('v3.0.0', '<')
+    - peer_with_router|default(false)
+
+- name: Calico | Create calico manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: calico-config, file: calico-config.yml, type: cm}
+    - {name: calico-node, file: calico-node.yml, type: ds}
+    - {name: calico, file: calico-node-sa.yml, type: sa}
+    - {name: calico, file: calico-cr.yml, type: clusterrole}
+    - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
+  register: calico_node_manifests
+  when:
+    - inventory_hostname in groups['kube-master']
+    - rbac_enabled or item.type not in rbac_resources
+
+- name: Calico | Create calico manifests for typha
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: calico, file: calico-typha.yml, type: typha}
+  register: calico_node_typha_manifest
+  when:
+    - inventory_hostname in groups['kube-master']
+    - typha_enabled and calico_datastore == "kdd"
+
+- name: Start Calico resources
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ calico_node_manifests.results }}"
+    - "{{ calico_node_kdd_manifest.results }}"
+    - "{{ calico_node_typha_manifest.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+
+- name: Wait for calico kubeconfig to be created
+  wait_for:
+    path: /etc/cni/net.d/calico-kubeconfig
+  when:
+    - inventory_hostname not in groups['kube-master']
+    - calico_datastore == "kdd"
+
 - name: Calico | Configure node asNumber for per node peering
   shell: >
    echo '{
@@ -209,7 +297,6 @@
    }}' | {{ bin_dir }}/calicoctl.sh {{ 'apply -f -' if calico_datastore == "kdd" else 'create --skip-exists -f -' }}
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
-  delegate_to: "{{ groups['kube-master'][0] }}"
   when:
     - calico_version is version('v3.0.0', '>=')
     - peer_with_router|default(false)
@@ -257,7 +344,6 @@
   delay: "{{ retry_stagger | random + 3 }}"
   with_items:
     - "{{ peers|selectattr('scope','undefined')|list|default([]) | union(peers|selectattr('scope','defined')|selectattr('scope','equalto', 'node')|list|default([])) }}"
-  delegate_to: "{{ groups['kube-master'][0] }}"
   when:
     - calico_version is version('v3.0.0', '>=')
     - peer_with_router|default(false)
@@ -280,46 +366,6 @@
     - peer_with_router|default(false)
     - inventory_hostname in groups['k8s-cluster']
 
-- name: Calico | Configure peering with router(s) at global scope
-  shell: >
-   echo '{
-   "apiVersion": "projectcalico.org/v3",
-   "kind": "BGPPeer",
-   "metadata": {
-      "name": "global-{{ item.router_id }}"
-   },
-   "spec": {
-      "asNumber": "{{ item.as }}",
-      "peerIP": "{{ item.router_id }}"
-   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  with_items:
-    - "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|list|default([]) }}"
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
-  when:
-    - calico_version | version_compare('v3.0.0', '>=')
-    - peer_with_router|default(false)
-    - inventory_hostname in groups['k8s-cluster']
-
-- name: Calico | Configure peering with router(s) at global scope (legacy)
-  shell: >
-   echo '{
-   "kind": "bgpPeer",
-   "spec": {"asNumber": "{{ item.as }}"},
-   "apiVersion": "v1",
-   "metadata": {"scope": "global", "peerIP": "{{ item.router_id }}"}
-   }'
-   | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  with_items: "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|default([]) }}"
-  run_once: true
-  when:
-    - calico_version is version('v3.0.0', '<')
-    - peer_with_router|default(false)
-    - inventory_hostname in groups['k8s-cluster']
 
 - name: Calico | Configure peering with route reflectors
   shell: >
@@ -338,7 +384,6 @@
   delay: "{{ retry_stagger | random + 3 }}"
   with_items:
     - "{{ groups['calico-rr'] | default([]) }}"
-  delegate_to: "{{ groups['kube-master'][0] }}"
   when:
     - calico_version is version('v3.0.0', '>=')
     - peer_with_calico_rr|default(false)
@@ -364,30 +409,3 @@
     - not calico_upgrade_enabled
     - peer_with_calico_rr|default(false)
     - hostvars[item]['cluster_id'] == cluster_id
-
-
-- name: Calico | Create calico manifests
-  template:
-    src: "{{ item.file }}.j2"
-    dest: "{{ kube_config_dir }}/{{ item.file }}"
-  with_items:
-    - {name: calico-config, file: calico-config.yml, type: cm}
-    - {name: calico-node, file: calico-node.yml, type: ds}
-    - {name: calico, file: calico-node-sa.yml, type: sa}
-    - {name: calico, file: calico-cr.yml, type: clusterrole}
-    - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
-  register: calico_node_manifests
-  when:
-    - inventory_hostname in groups['kube-master']
-    - rbac_enabled or item.type not in rbac_resources
-
-- name: Calico | Create calico manifests for typha
-  template:
-    src: "{{ item.file }}.j2"
-    dest: "{{ kube_config_dir }}/{{ item.file }}"
-  with_items:
-    - {name: calico, file: calico-typha.yml, type: typha}
-  register: calico_node_typha_manifest
-  when:
-    - inventory_hostname in groups['kube-master']
-    - typha_enabled and calico_datastore == "kdd"
diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml
index 881c2eb5f..75679a8e4 100644
--- a/roles/network_plugin/calico/tasks/main.yml
+++ b/roles/network_plugin/calico/tasks/main.yml
@@ -3,11 +3,11 @@
 
 - import_tasks: pre.yml
 
-- import_tasks: upgrade.yml
+- include_tasks: upgrade.yml
   when:
     - calico_upgrade_enabled
     - calico_upgrade_needed
+    - inventory_hostname in groups['kube-master']
   run_once: yes
-  delegate_to: "{{ groups['kube-master'][0] }}"
 
 - include_tasks: install.yml
diff --git a/roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2 b/roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2
index c795dcb05..e6e4ec6e8 100644
--- a/roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2
+++ b/roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2
@@ -1,6 +1,8 @@
 #!/bin/bash
 DATASTORE_TYPE=kubernetes \
 {% if inventory_hostname in groups['kube-master'] %}
-KUBECONFIG={{ kube_config_dir }}/admin.conf \
+KUBECONFIG=/etc/kubernetes/admin.conf \
+{% else %}
+KUBECONFIG=/etc/cni/net.d/calico-kubeconfig \
 {% endif %}
 {{ bin_dir }}/calicoctl "$@"