From 35682b52281fc5498ac3bce851503728ee8fee12 Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Thu, 6 Aug 2020 09:14:55 +0200 Subject: [PATCH] Fix cilium strict kube proxy replacement in HA (#6473) * Update the cilium svc proxy test to HA mode Signed-off-by: Arthur Outhenin-Chalandre * Fix cilium strict kube-proxy in HA Signed-off-by: Arthur Outhenin-Chalandre * Add a single global endpoint variable Signed-off-by: Arthur Outhenin-Chalandre * Add cilium docs about kube-proxy replacement Signed-off-by: Arthur Outhenin-Chalandre * Fix issues in docs Signed-off-by: Arthur Outhenin-Chalandre --- docs/cilium.md | 13 +++++++++++++ roles/kubespray-defaults/defaults/main.yaml | 6 ++++++ .../cilium/templates/cilium-deploy.yml.j2 | 6 ++++++ .../cilium/templates/cilium-ds.yml.j2 | 6 ++++++ tests/files/packet_debian10-cilium-svc-proxy.yml | 2 +- 5 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 docs/cilium.md diff --git a/docs/cilium.md b/docs/cilium.md new file mode 100644 index 000000000..ea4294a94 --- /dev/null +++ b/docs/cilium.md @@ -0,0 +1,13 @@ +# Cilium + +## Kube-proxy replacement with Cilium + +Cilium can run without kube-proxy by setting `cilium_kube_proxy_replacement` +to `strict`. + +Without kube-proxy, cilium needs to know the address of the kube-apiserver +and this must be set globally for all cilium components (agents and operators). +Hence, in this configuration in Kubespray, Cilium will always contact +the external loadbalancer (even from a node in the control plane) +and if there is no external load balancer It will ignore any local load +balancer deployed by Kubespray and **only contacts the first master**. diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 5b7e5cd12..e30575f0c 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -448,6 +448,12 @@ loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" loadbalancer_apiserver_type: "nginx" # applied if only external loadbalancer_apiserver is defined, otherwise ignored apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" +kube_apiserver_global_endpoint: |- + {% if loadbalancer_apiserver is defined -%} + https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} + {%- else -%} + https://{{ first_kube_master }}:{{ kube_apiserver_port }} + {%- endif %} kube_apiserver_endpoint: |- {% if loadbalancer_apiserver is defined -%} https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} diff --git a/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 index 18fdad7bc..bf89286e2 100644 --- a/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 @@ -84,6 +84,12 @@ spec: key: AWS_DEFAULT_REGION name: cilium-aws optional: true +{% if cilium_kube_proxy_replacement == 'strict' %} + - name: KUBERNETES_SERVICE_HOST + value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" + - name: KUBERNETES_SERVICE_PORT + value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" +{% endif %} image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}" imagePullPolicy: {{ k8s_image_pull_policy }} name: cilium-operator diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 index 9393c6f37..dbf26ab0f 100755 --- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -43,6 +43,12 @@ spec: fieldPath: metadata.namespace - name: CILIUM_CLUSTERMESH_CONFIG value: /var/lib/cilium/clustermesh/ +{% if cilium_kube_proxy_replacement == 'strict' %} + - name: KUBERNETES_SERVICE_HOST + value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" + - name: KUBERNETES_SERVICE_PORT + value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" +{% endif %} image: "{{cilium_image_repo}}:{{cilium_image_tag}}" imagePullPolicy: {{ k8s_image_pull_policy }} resources: diff --git a/tests/files/packet_debian10-cilium-svc-proxy.yml b/tests/files/packet_debian10-cilium-svc-proxy.yml index eb163332c..9565a6ab6 100644 --- a/tests/files/packet_debian10-cilium-svc-proxy.yml +++ b/tests/files/packet_debian10-cilium-svc-proxy.yml @@ -1,7 +1,7 @@ --- # Instance settings cloud_image: debian-10 -mode: separate +mode: ha # Kubespray settings kube_network_plugin: cilium