feat: upgrade azuredisk csi to v1.10.0 (#8432)

Signed-off-by: Cyril Corbon <corboncyril@gmail.com>

roles/download/defaults/main.yml

@@ -673,15 +673,14 @@ aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-drive
 aws_ebs_csi_plugin_image_tag: "v0.5.0"
 
 azure_csi_image_repo: "mcr.microsoft.com/oss/kubernetes-csi"
-azure_csi_provisioner_image_tag: "v1.5.0"
-azure_csi_attacher_image_tag: "v1.2.0"
-azure_csi_cluster_registrar_image_tag: "v1.0.1"
-azure_csi_node_registrar_image_tag: "v1.1.0"
-azure_csi_snapshotter_image_tag: "v2.0.0"
-azure_csi_resizer_image_tag: "v0.3.0"
-azure_csi_livenessprobe_image_tag: "v1.1.0"
+azure_csi_provisioner_image_tag: "v2.2.2"
+azure_csi_attacher_image_tag: "v3.3.0"
+azure_csi_resizer_image_tag: "v1.3.0"
+azure_csi_livenessprobe_image_tag: "v2.5.0"
+azure_csi_node_registrar_image_tag: "v2.4.0"
+azure_csi_snapshotter_image_tag: "v3.0.3"
 azure_csi_plugin_image_repo: "mcr.microsoft.com/k8s/csi"
-azure_csi_plugin_image_tag: "v0.7.0"
+azure_csi_plugin_image_tag: "v1.10.0"
 
 gcp_pd_csi_image_repo: "gke.gcr.io"
 gcp_pd_csi_driver_image_tag: "v0.7.0-gke.0"

roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml

@@ -1,4 +1,6 @@
 ---
 azure_csi_use_instance_metadata: true
-azure_csi_controller_replicas: 1
+azure_csi_controller_replicas: 2
 azure_csi_plugin_image_tag: latest
+azure_csi_controller_affinity: {}
+azure_csi_node_affinity: {}

roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2

@@ -71,7 +71,10 @@ rules:
     verbs: ["get", "list", "watch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "list", "watch", "create", "update", "patch"]
@@ -128,15 +131,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: azuredisk-external-snapshotter-role
 rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
@@ -150,14 +144,11 @@ rules:
     resources: ["volumesnapshotcontents"]
     verbs: ["create", "get", "list", "watch", "update", "delete"]
   - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
   - apiGroups: ["apiextensions.k8s.io"]
     resources: ["customresourcedefinitions"]
     verbs: ["create", "list", "watch", "delete"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents/status"]
-    verbs: ["update"]
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "watch", "list", "delete", "update", "create"]
@@ -197,6 +188,9 @@ rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -210,3 +204,27 @@ roleRef:
   kind: ClusterRole
   name: azuredisk-external-resizer-role
   apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-controller-secret-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-controller-secret-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-azuredisk-controller-secret-role
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2

@@ -24,19 +24,23 @@ spec:
           effect: "NoSchedule"
         - key: "node-role.kubernetes.io/control-plane"
           effect: "NoSchedule"
+{% if azure_csi_controller_affinity %}
+      affinity:
+        {{ azure_csi_controller_affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
       containers:
         - name: csi-provisioner
           image: {{ azure_csi_image_repo }}/csi-provisioner:{{ azure_csi_provisioner_image_tag }}
           imagePullPolicy: {{ k8s_image_pull_policy }}
           args:
-            - "--provisioner=disk.csi.azure.com"
             - "--feature-gates=Topology=true"
             - "--csi-address=$(ADDRESS)"
-            - "--connection-timeout=15s"
-            - "--v=5"
-            - "--timeout=120s"
-            - "--enable-leader-election"
-            - "--leader-election-type=leases"
+            - "--v=2"
+            - "--timeout=15s"
+            - "--leader-election"
+            - "--worker-threads=40"
+            - "--extra-create-metadata=true"
+            - "--strict-topology=true"
           env:
             - name: ADDRESS
               value: /csi/csi.sock
@@ -45,8 +49,7 @@ spec:
               name: socket-dir
           resources:
             limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 500Mi
             requests:
               cpu: 10m
               memory: 20Mi
@@ -54,40 +57,20 @@ spec:
           image: {{ azure_csi_image_repo }}/csi-attacher:{{ azure_csi_attacher_image_tag }}
           imagePullPolicy: {{ k8s_image_pull_policy }}
           args:
-            - "-v=5"
+            - "-v=2"
             - "-csi-address=$(ADDRESS)"
-            - "-timeout=120s"
+            - "-timeout=600s"
             - "-leader-election"
-            - "-leader-election-type=leases"
+            - "-worker-threads=500"
           env:
             - name: ADDRESS
               value: /csi/csi.sock
           volumeMounts:
-            - mountPath: /csi
-              name: socket-dir
+          - mountPath: /csi
+            name: socket-dir
           resources:
             limits:
-              cpu: 200m
-              memory: 200Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-        - name: cluster-driver-registrar
-          image: {{ azure_csi_image_repo }}/csi-cluster-driver-registrar:{{ azure_csi_cluster_registrar_image_tag }}
-          args:
-            - --csi-address=$(ADDRESS)
-            - --driver-requires-attachment=true
-            - --v=5
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-          resources:
-            limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 500Mi
             requests:
               cpu: 10m
               memory: 20Mi
@@ -96,7 +79,7 @@ spec:
           args:
             - "-csi-address=$(ADDRESS)"
             - "-leader-election"
-            - "--v=5"
+            - "-v=2"
           env:
             - name: ADDRESS
               value: /csi/csi.sock
@@ -105,8 +88,7 @@ spec:
               mountPath: /csi
           resources:
             limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 100Mi
             requests:
               cpu: 10m
               memory: 20Mi
@@ -114,8 +96,10 @@ spec:
           image: {{ azure_csi_image_repo }}/csi-resizer:{{ azure_csi_resizer_image_tag }}
           args:
             - "-csi-address=$(ADDRESS)"
-            - "-v=5"
+            - "-v=2"
             - "-leader-election"
+            - '-handle-volume-inuse-error=false'
+            - "-timeout=60s"
           env:
             - name: ADDRESS
               value: /csi/csi.sock
@@ -124,8 +108,7 @@ spec:
               mountPath: /csi
           resources:
             limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 500Mi
             requests:
               cpu: 10m
               memory: 20Mi
@@ -133,16 +116,15 @@ spec:
           image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }}
           args:
             - --csi-address=/csi/csi.sock
-            - --connection-timeout=3s
+            - --probe-timeout=3s
             - --health-port=29602
-            - --v=5
+            - --v=2
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
           resources:
             limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 100Mi
             requests:
               cpu: 10m
               memory: 20Mi
@@ -152,7 +134,11 @@ spec:
           args:
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--nodeid=$(KUBE_NODE_NAME)"
+            - "--metrics-address=0.0.0.0:29604"
+            - "--disable-avset-nodes=true"
+            - "--drivername=disk.csi.azure.com"
+            - "--cloud-config-secret-name=cloud-config"
+            - "--cloud-config-secret-namespace=kube-system"
           ports:
             - containerPort: 29602
               name: healthz
@@ -184,8 +170,7 @@ spec:
               name: msi
           resources:
             limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 500Mi
             requests:
               cpu: 10m
               memory: 20Mi
@@ -195,6 +180,3 @@ spec:
         - name: azure-cred
           secret:
             secretName: cloud-config
-        - name: msi
-          hostPath:
-            path: /var/lib/waagent/ManagedIdentity-Settings

roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2

@@ -0,0 +1,30 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-azuredisk-node-sa
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-node-secret-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-node-secret-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-node-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-azuredisk-node-secret-role
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2

@@ -5,6 +5,10 @@ metadata:
   name: csi-azuredisk-node
   namespace: kube-system
 spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
   selector:
     matchLabels:
       app: csi-azuredisk-node
@@ -14,9 +18,17 @@ spec:
         app: csi-azuredisk-node
     spec:
       hostNetwork: true
+      dnsPolicy: Default
+      serviceAccountName: csi-azuredisk-node-sa
       nodeSelector:
         kubernetes.io/os: linux
+{% if azure_csi_node_affinity %}
+      affinity:
+        {{ azure_csi_node_affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
       priorityClassName: system-node-critical
+      tolerations:
+        - operator: Exists
       containers:
         - name: liveness-probe
           volumeMounts:
@@ -26,13 +38,12 @@ spec:
           imagePullPolicy: {{ k8s_image_pull_policy }}
           args:
             - --csi-address=/csi/csi.sock
-            - --connection-timeout=3s
+            - --probe-timeout=3s
             - --health-port=29603
-            - --v=5
+            - --v=2
           resources:
             limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 100Mi
             requests:
               cpu: 10m
               memory: 20Mi
@@ -41,11 +52,15 @@ spec:
           args:
             - --csi-address=$(ADDRESS)
             - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
-            - --v=5
-          lifecycle:
-            preStop:
-              exec:
-                command: ["/bin/sh", "-c", "rm -rf /registration/disk.csi.azure.com-reg.sock /csi/csi.sock"]
+            - --v=2
+          livenessProbe:
+            exec:
+              command:
+                - /csi-node-driver-registrar
+                - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+                - --mode=kubelet-registration-probe
+            initialDelaySeconds: 30
+            timeoutSeconds: 15
           env:
             - name: ADDRESS
               value: /csi/csi.sock
@@ -58,8 +73,7 @@ spec:
               mountPath: /registration
           resources:
             limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 100Mi
             requests:
               cpu: 10m
               memory: 20Mi
@@ -71,6 +85,11 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodeid=$(KUBE_NODE_NAME)"
             - "--metrics-address=0.0.0.0:29605"
+            - "--enable-perf-optimization=true"
+            - "--drivername=disk.csi.azure.com"
+            - "--volume-attach-limit=-1"
+            - "--cloud-config-secret-name=cloud-config"
+            - "--cloud-config-secret-namespace=kube-system"
           ports:
             - containerPort: 29603
               name: healthz
@@ -106,10 +125,6 @@ spec:
               name: mountpoint-dir
             - mountPath: /etc/kubernetes/
               name: azure-cred
-              readOnly: true
-            - mountPath: /var/lib/waagent/ManagedIdentity-Settings
-              readOnly: true
-              name: msi
             - mountPath: /dev
               name: device-dir
             - mountPath: /sys/bus/scsi/devices
@@ -118,7 +133,6 @@ spec:
               name: scsi-host-dir
           resources:
             limits:
-              cpu: 200m
               memory: 200Mi
             requests:
               cpu: 10m
@@ -136,12 +150,10 @@ spec:
             path: /var/lib/kubelet/plugins_registry/
             type: DirectoryOrCreate
           name: registration-dir
-        - name: azure-cred
-          secret:
-            secretName: cloud-config
         - hostPath:
-            path: /var/lib/waagent/ManagedIdentity-Settings
-          name: msi
+            path: /etc/kubernetes/
+            type: DirectoryOrCreate
+          name: azure-cred
         - hostPath:
             path: /dev
             type: Directory

roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2

@@ -1,42 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  name: csinodeinfos.csi.storage.k8s.io
-spec:
-  group: csi.storage.k8s.io
-  scope: Cluster
-  names:
-    kind: CSINodeInfo
-    plural: csinodeinfos
-  versions:
-  - name: v1alpha1
-    served: true
-    storage: true
-    schema:
-      openAPIV3Schema:
-        type: object
-        properties:
-          csiDrivers:
-            description: List of CSI drivers running on the node and their properties.
-            items:
-              properties:
-                driver:
-                  description: The CSI driver that this object refers to.
-                  type: string
-                nodeID:
-                  description: The node from the driver point of view.
-                  type: string
-                topologyKeys:
-                  description: List of keys supported by the driver.
-                  items:
-                    type: string
-                  type: array
-            type: array
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []