From 4a03d13d083c303a36b323ad6a9e70f37946b4dc Mon Sep 17 00:00:00 2001 From: jeremy-thuon <18218996+jeremythuon@users.noreply.github.com> Date: Mon, 10 Apr 2023 07:07:15 +0200 Subject: [PATCH] [cilium] fix rbac and upgrade hubble v0.11.0 (#3) (#9959) * [cilium] fix rbac and upgrade hubble v0.11.0 (#3) * [cilium] fix rbac for LB bgp ipam * [cilium] Upgrade Hubble to v0.11.0 and add mTLS between Hubble UI and Hubble Relay * fix dns domain hubble for tls --------- Co-authored-by: Thuon Jeremy * Fix blank line --------- Co-authored-by: Thuon Jeremy --- roles/download/defaults/main.yml | 4 +-- roles/network_plugin/cilium/defaults/main.yml | 17 ++++++++++ .../templates/cilium-operator/cr.yml.j2 | 3 ++ .../cilium/templates/hubble/cronjob.yml.j2 | 17 +++------- .../cilium/templates/hubble/deploy.yml.j2 | 32 +++++++++++++++++++ .../cilium/templates/hubble/job.yml.j2 | 17 +++------- .../cilium/templates/hubble/service.yml.j2 | 4 +++ 7 files changed, 66 insertions(+), 28 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 76a8756c1..9528d4605 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -1038,9 +1038,9 @@ cilium_hubble_relay_image_tag: "{{ cilium_version }}" cilium_hubble_certgen_image_repo: "{{ quay_image_repo }}/cilium/certgen" cilium_hubble_certgen_image_tag: "v0.1.8" cilium_hubble_ui_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui" -cilium_hubble_ui_image_tag: "v0.9.2" +cilium_hubble_ui_image_tag: "v0.11.0" cilium_hubble_ui_backend_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui-backend" -cilium_hubble_ui_backend_image_tag: "v0.9.2" +cilium_hubble_ui_backend_image_tag: "v0.11.0" cilium_hubble_envoy_image_repo: "{{ docker_image_repo }}/envoyproxy/envoy" cilium_hubble_envoy_image_tag: "v1.22.5" kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn" diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml index 2be23f847..29dd08350 100644 --- a/roles/network_plugin/cilium/defaults/main.yml +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -273,3 +273,20 @@ cilium_rolling_restart_wait_retries_delay_seconds: 10 cilium_agent_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9962', '9090') }}" cilium_operator_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9963', '6942') }}" cilium_hubble_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9965', '9091') }}" + +# Cilium certgen args for generate certificate for hubble mTLS +cilium_certgen_args: + cilium-namespace: kube-system + ca-reuse-secret: true + ca-secret-name: hubble-ca-secret + ca-generate: true + ca-validity-duration: 94608000s + hubble-server-cert-generate: true + hubble-server-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io' + hubble-server-cert-validity-duration: 94608000s + hubble-server-cert-secret-name: hubble-server-certs + hubble-relay-client-cert-generate: true + hubble-relay-client-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io' + hubble-relay-client-cert-validity-duration: 94608000s + hubble-relay-client-cert-secret-name: hubble-relay-client-certs + hubble-relay-server-cert-generate: false diff --git a/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 index 8a40a6641..044695022 100644 --- a/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 @@ -54,6 +54,7 @@ rules: - services/status verbs: - update + - patch - apiGroups: - "" resources: @@ -92,6 +93,8 @@ rules: {% endif %} {% if cilium_version | regex_replace('v') is version('1.12', '>=') %} - ciliumbgploadbalancerippools + - ciliumloadbalancerippools + - ciliumloadbalancerippools/status - ciliumbgppeeringpolicies - ciliumenvoyconfigs {% endif %} diff --git a/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2 b/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2 index 1ec1f07d5..8010c5252 100644 --- a/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2 +++ b/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2 @@ -29,19 +29,10 @@ spec: # line args instead of via config map. This allows users to inspect # the values used in past runs by inspecting the completed pod. args: - - "--cilium-namespace=kube-system" - - "--ca-reuse-secret=true" - - "--ca-secret-name=hubble-ca-secret" - - "--ca-generate=true" - - "--ca-validity-duration=94608000s" - - "--hubble-server-cert-generate=true" - - "--hubble-server-cert-common-name=*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io" - - "--hubble-server-cert-validity-duration=94608000s" - - "--hubble-server-cert-secret-name=hubble-server-certs" - - "--hubble-relay-client-cert-generate=true" - - "--hubble-relay-client-cert-validity-duration=94608000s" - - "--hubble-relay-client-cert-secret-name=hubble-relay-client-certs" - - "--hubble-relay-server-cert-generate=false" + {% for key, value in cilium_certgen_args.items() -%} + - "--{{ key }}={{ value }}" + {% endfor %} + hostNetwork: true restartPolicy: OnFailure ttlSecondsAfterFinished: 1800 diff --git a/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2 b/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2 index 2cfa25eb3..d9cc4973a 100644 --- a/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2 +++ b/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2 @@ -138,8 +138,28 @@ spec: env: - name: EVENTS_SERVER_PORT value: "8090" + {% if cilium_hubble_tls_generate -%} + - name: TLS_TO_RELAY_ENABLED + value: "true" + - name: FLOWS_API_ADDR + value: "hubble-relay:443" + - name: TLS_RELAY_SERVER_NAME + value: ui.{{ cilium_cluster_name }}.hubble-grpc.cilium.io + - name: TLS_RELAY_CA_CERT_FILES + value: /var/lib/hubble-ui/certs/hubble-server-ca.crt + - name: TLS_RELAY_CLIENT_CERT_FILE + value: /var/lib/hubble-ui/certs/client.crt + - name: TLS_RELAY_CLIENT_KEY_FILE + value: /var/lib/hubble-ui/certs/client.key + {% else -%} - name: FLOWS_API_ADDR value: "hubble-relay:80" + {% endif %} + + volumeMounts: + - name: tls + mountPath: /var/lib/hubble-ui/certs + readOnly: true ports: - containerPort: 8090 name: grpc @@ -150,5 +170,17 @@ spec: defaultMode: 420 name: hubble-ui-nginx name: hubble-ui-nginx-conf + - projected: + sources: + - secret: + name: hubble-relay-client-certs + items: + - key: ca.crt + path: hubble-server-ca.crt + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key + name: tls - emptyDir: {} name: tmp-dir diff --git a/roles/network_plugin/cilium/templates/hubble/job.yml.j2 b/roles/network_plugin/cilium/templates/hubble/job.yml.j2 index 544006018..9ad3ae318 100644 --- a/roles/network_plugin/cilium/templates/hubble/job.yml.j2 +++ b/roles/network_plugin/cilium/templates/hubble/job.yml.j2 @@ -25,19 +25,10 @@ spec: # line args instead of via config map. This allows users to inspect # the values used in past runs by inspecting the completed pod. args: - - "--cilium-namespace=kube-system" - - "--ca-reuse-secret=true" - - "--ca-secret-name=hubble-ca-secret" - - "--ca-generate=true" - - "--ca-validity-duration=94608000s" - - "--hubble-server-cert-generate=true" - - "--hubble-server-cert-common-name=*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io" - - "--hubble-server-cert-validity-duration=94608000s" - - "--hubble-server-cert-secret-name=hubble-server-certs" - - "--hubble-relay-client-cert-generate=true" - - "--hubble-relay-client-cert-validity-duration=94608000s" - - "--hubble-relay-client-cert-secret-name=hubble-relay-client-certs" - - "--hubble-relay-server-cert-generate=false" + {% for key, value in cilium_certgen_args.items() -%} + - "--{{ key }}={{ value }}" + {% endfor %} + hostNetwork: true restartPolicy: OnFailure ttlSecondsAfterFinished: 1800 diff --git a/roles/network_plugin/cilium/templates/hubble/service.yml.j2 b/roles/network_plugin/cilium/templates/hubble/service.yml.j2 index e3e882d9d..f1df0ebb2 100644 --- a/roles/network_plugin/cilium/templates/hubble/service.yml.j2 +++ b/roles/network_plugin/cilium/templates/hubble/service.yml.j2 @@ -58,7 +58,11 @@ spec: k8s-app: hubble-relay ports: - protocol: TCP + {% if cilium_hubble_tls_generate -%} + port: 443 + {% else -%} port: 80 + {% endif -%} targetPort: 4245 --- # Source: cilium/templates/hubble-ui-service.yaml