add ingress nginx webhook (#9033)

* add ingress nginx webhook

* fix ingress nginx template

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -123,6 +123,7 @@ ingress_publish_status_address: ""
 #   - --default-ssl-certificate=default/foo-tls
 # ingress_nginx_termination_grace_period_seconds: 300
 # ingress_nginx_class: nginx
+# ingress_nginx_webhook_enabled: false
 
 # ALB ingress controller deployment
 ingress_alb_enabled: false

roles/download/defaults/main.yml

@@ -893,6 +893,8 @@ local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-p
 local_path_provisioner_image_tag: "v0.0.21"
 ingress_nginx_controller_image_repo: "{{ kube_image_repo }}/ingress-nginx/controller"
 ingress_nginx_controller_image_tag: "v1.2.1"
+ingress_nginx_kube_webhook_certgen_imae_repo: "{{ kube_image_repo }}/ingress-nginx/kube-webhook-certgen"
+ingress_nginx_kube_webhook_certgen_imae_tag: "v1.1.1"
 alb_ingress_image_repo: "{{ docker_image_repo }}/amazon/aws-alb-ingress-controller"
 alb_ingress_image_tag: "v1.1.9"
 cert_manager_version: "v1.8.2"

roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml

@@ -14,3 +14,5 @@ ingress_nginx_configmap_udp_services: {}
 ingress_nginx_extra_args: []
 ingress_nginx_termination_grace_period_seconds: 300
 # ingress_nginx_class: nginx
+ingress_nginx_webhook_enabled: false
+ingress_nginx_webhook_job_ttl: 1800

roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml

@@ -25,12 +25,25 @@
       - { name: ds-ingress-nginx-controller, file: ds-ingress-nginx-controller.yml, type: ds }
     ingress_nginx_templates_for_psp:
       - { name: psp-ingress-nginx, file: psp-ingress-nginx.yml, type: podsecuritypolicy }
+    ingress_nginx_templates_for_webhook:
+      - { name: admission-webhook-configuration, file: admission-webhook-configuration.yml, type: sa }
+      - { name: sa-admission-webhook, file: sa-admission-webhook.yml, type: sa }
+      - { name: clusterrole-admission-webhook, file: clusterrole-admission-webhook.yml, type: clusterrole }
+      - { name: clusterrolebinding-admission-webhook, file: clusterrolebinding-admission-webhook.yml, type: clusterrolebinding }
+      - { name: role-admission-webhook, file: role-admission-webhook.yml, type: role }
+      - { name: rolebinding-admission-webhook, file: rolebinding-admission-webhook.yml, type: rolebinding }
+      - { name: admission-webhook-job, file: admission-webhook-job.yml, type: job }
 
 - name: NGINX Ingress Controller | Append extra templates to NGINX Ingress Templates list for PodSecurityPolicy
   set_fact:
     ingress_nginx_templates: "{{ ingress_nginx_templates_for_psp + ingress_nginx_templates }}"
   when: podsecuritypolicy_enabled
 
+- name: NGINX Ingress Controller | Append extra templates to NGINX Ingress Templates list for PodSecurityPolicy
+  set_fact:
+    ingress_nginx_templates: "{{ ingress_nginx_templates + ingress_nginx_templates_for_webhook }}"
+  when: ingress_nginx_webhook_enabled
+
 - name: NGINX Ingress Controller | Create manifests
   template:
     src: "{{ item.file }}.j2"

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-configuration.yml.j2

@@ -0,0 +1,29 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  name: ingress-nginx-admission
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: ingress-nginx-controller-admission
+      namespace: {{ ingress_nginx_namespace }}
+      path: /networking/v1/ingresses
+  failurePolicy: Fail
+  matchPolicy: Equivalent
+  name: validate.nginx.ingress.kubernetes.io
+  rules:
+  - apiGroups:
+    - networking.k8s.io
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - ingresses
+  sideEffects: None

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-job.yml.j2

@@ -0,0 +1,86 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  name: ingress-nginx-admission-create
+  namespace: {{ ingress_nginx_namespace }}
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+      name: ingress-nginx-admission-create
+    spec:
+      containers:
+      - args:
+        - create
+        - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
+        - --namespace=$(POD_NAMESPACE)
+        - --secret-name=ingress-nginx-admission
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: "{{ ingress_nginx_kube_webhook_certgen_imae_repo }}:{{ ingress_nginx_kube_webhook_certgen_imae_tag }}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        name: create
+        securityContext:
+          allowPrivilegeEscalation: false
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: OnFailure
+      securityContext:
+        fsGroup: 2000
+        runAsNonRoot: true
+        runAsUser: 2000
+      serviceAccountName: ingress-nginx-admission
+  ttlSecondsAfterFinished: {{ ingress_nginx_webhook_job_ttl }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  name: ingress-nginx-admission-patch
+  namespace: {{ ingress_nginx_namespace }}
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+      name: ingress-nginx-admission-patch
+    spec:
+      containers:
+      - args:
+        - patch
+        - --webhook-name=ingress-nginx-admission
+        - --namespace=$(POD_NAMESPACE)
+        - --patch-mutating=false
+        - --secret-name=ingress-nginx-admission
+        - --patch-failure-policy=Fail
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: "{{ ingress_nginx_kube_webhook_certgen_imae_repo }}:{{ ingress_nginx_kube_webhook_certgen_imae_tag }}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        name: patch
+        securityContext:
+          allowPrivilegeEscalation: false
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: OnFailure
+      securityContext:
+        fsGroup: 2000
+        runAsNonRoot: true
+        runAsUser: 2000
+      serviceAccountName: ingress-nginx-admission
+  ttlSecondsAfterFinished: {{ ingress_nginx_webhook_job_ttl }}

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-admission-webhook.yml.j2

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  name: ingress-nginx-admission
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingwebhookconfigurations
+  verbs:
+  - get
+  - update

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2

@@ -28,4 +28,3 @@ rules:
   - apiGroups: ["networking.k8s.io"]
     resources: ["ingressclasses"]
     verbs: ["get", "list", "watch"]
-

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-admission-webhook.yml.j2

@@ -0,0 +1,16 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  name: ingress-nginx-admission
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx-admission
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx-admission
+    namespace: {{ ingress_nginx_namespace }}

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2

@@ -65,6 +65,11 @@ spec:
 {% for extra_arg in ingress_nginx_extra_args %}
             - {{ extra_arg }}
 {% endfor %}
+{% if ingress_nginx_webhook_enabled %}
+            - --validating-webhook=:8443
+            - --validating-webhook-certificate=/usr/local/certificates/cert
+            - --validating-webhook-key=/usr/local/certificates/key
+{% endif %}
           securityContext:
             capabilities:
                 drop:
@@ -96,6 +101,11 @@ spec:
               containerPort: 10254
 {% if not ingress_nginx_host_network %}
               hostPort: {{ ingress_nginx_metrics_port }}
+{% endif %}
+{% if ingress_nginx_webhook_enabled %}
+            - name: webhook
+              containerPort: 8443
+              protocol: TCP
 {% endif %}
           livenessProbe:
             failureThreshold: 3
@@ -118,3 +128,15 @@ spec:
             timeoutSeconds: 5
             successThreshold: 1
             failureThreshold: 3
+{% if ingress_nginx_webhook_enabled %}
+          volumeMounts:
+            - mountPath: /usr/local/certificates/
+              name: webhook-cert
+              readOnly: true
+{% endif %}
+{% if ingress_nginx_webhook_enabled %}
+      volumes:
+        - name: webhook-cert
+          secret:
+            secretName: ingress-nginx-admission
+{% endif %}

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.ym.j2

@@ -0,0 +1,17 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  name: ingress-nginx-admission
+  namespace: {{ ingress_nginx_namespace }}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.yml.j2

@@ -0,0 +1,17 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  name: ingress-nginx-admission
+  namespace: {{ ingress_nginx_namespace }}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-admission-webhook.yml.j2

@@ -0,0 +1,17 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  name: ingress-nginx-admission
+  namespace: {{ ingress_nginx_namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx-admission
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx-admission
+  namespace: {{ ingress_nginx_namespace }}

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-admission-webhook.yml.j2

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ingress-nginx-admission
+  namespace: {{ ingress_nginx_namespace }}
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx

tests/files/packet_centos7-flannel-addons-ha.yml

@@ -16,6 +16,8 @@ etcd_events_cluster_enabled: true
 local_volume_provisioner_enabled: true
 kube_encrypt_secret_data: true
 ingress_nginx_enabled: true
+ingress_nginx_webhook_enabled: true
+ingress_nginx_webhook_job_ttl: 30
 cert_manager_enabled: true
 # Disable as health checks are still unstable and slow to respond.
 metrics_server_enabled: false

tests/files/packet_ubuntu18-flannel-ha.yml

@@ -14,6 +14,8 @@ etcd_events_cluster_enabled: true
 local_volume_provisioner_enabled: true
 kube_encrypt_secret_data: true
 ingress_nginx_enabled: true
+ingress_nginx_webhook_enabled: true
+ingress_nginx_webhook_job_ttl: 30
 cert_manager_enabled: true
 # Disable as health checks are still unstable and slow to respond.
 metrics_server_enabled: false