Fix ciliums hubble relay configuration (#9876)

* Fix ciliums hubble relay configuration

* Fixed the tls from code review

* Updated to dna_domain instead of hardcoding

roles/network_plugin/cilium/templates/hubble/config.yml.j2

@@ -1,5 +1,5 @@
 ---
-# Source: cilium/templates/hubble-relay-configmap.yaml
+# Source: cilium helm chart: cilium/templates/hubble-relay/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -7,12 +7,13 @@ metadata:
   namespace: kube-system
 data:
   config.yaml: |
-    peer-service: unix:///var/run/cilium/hubble.sock
+    peer-service: "hubble-peer.kube-system.svc.{{ dns_domain }}:443"
     listen-address: :4245
-    dial-timeout:
-    retry-timeout:
-    sort-buffer-len-max:
-    sort-buffer-drain-timeout:
+    metrics-listen-address: ":9966"
+    dial-timeout: 
+    retry-timeout: 
+    sort-buffer-len-max: 
+    sort-buffer-drain-timeout: 
     tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt
     tls-client-key-file: /var/lib/hubble-relay/tls/client.key
     tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt

roles/network_plugin/cilium/templates/hubble/deploy.yml.j2

@@ -83,9 +83,6 @@ spec:
                   path: client.crt
                 - key: tls.key
                   path: client.key
-          - configMap:
-              name: hubble-ca-cert
-              items:
                 - key: ca.crt
                   path: hubble-server-ca.crt
         name: tls

roles/network_plugin/cilium/templates/hubble/service.yml.j2

@@ -21,6 +21,27 @@ spec:
     targetPort: hubble-metrics
   selector:
     k8s-app: cilium
+---
+# Source: cilium/templates/hubble-relay/metrics-service.yaml
+# We use a separate service from hubble-relay which can be exposed externally
+kind: Service
+apiVersion: v1
+metadata:
+  name: hubble-relay-metrics
+  namespace: kube-system
+  labels:
+    k8s-app: hubble-relay
+spec:
+  clusterIP: None
+  type: ClusterIP
+  selector:
+    k8s-app: hubble-relay
+  ports:
+  - name: metrics
+    port: 9966
+    protocol: TCP
+    targetPort: prometheus
+
 {% endif %}
 ---
 # Source: cilium/templates/hubble-relay-service.yaml
@@ -56,3 +77,22 @@ spec:
       port: 80
       targetPort: 8081
   type: ClusterIP
+---
+# Source: cilium/templates/hubble/peer-service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: hubble-peer
+  namespace: kube-system
+  labels:
+    k8s-app: cilium
+spec:
+  selector:
+    k8s-app: cilium
+  ports:
+  - name: peer-service
+    port: 443
+    protocol: TCP
+    targetPort: 4244
+  internalTrafficPolicy: Local
+