Add suport of Vsphere CSI driver 2.X versions (#7480)

docs/vsphere-csi.md

@@ -17,6 +17,7 @@ You need to source the vSphere credentials you use to deploy your machines that
 | external_vsphere_password                   | TRUE     | string  |                            |                           | Password for vCenter                                           |
 | external_vsphere_datacenter                 | TRUE     | string  |                            |                           | Datacenter name to use                                         |
 | external_vsphere_kubernetes_cluster_id      | TRUE     | string  |                            | "kubernetes-cluster-id"   | Kubernetes cluster ID to use                                   |
+| external_vsphere_version          | TRUE     | string  |                            | "6.7u3"                  | Vmware Vsphere version where located all VMs                                   |
 | vsphere_cloud_controller_image_tag          | TRUE     | string  |                            | "latest"                  | Kubernetes cluster ID to use                                   |
 | vsphere_syncer_image_tag                    | TRUE     | string  |                            | "v1.0.2"                  | Syncer image tag to use                                        |
 | vsphere_csi_attacher_image_tag              | TRUE     | string  |                            | "v1.1.1"                  | CSI attacher image tag to use                                  |
@@ -26,6 +27,7 @@ You need to source the vSphere credentials you use to deploy your machines that
 | vsphere_csi_provisioner_image_tag           | TRUE     | string  |                            | "v1.2.2"                  | CSI provisioner image tag to use                               |
 | vsphere_csi_node_driver_registrar_image_tag | TRUE     | string  |                            | "v1.1.0"                  | CSI node driver registrat image tag to use                     |
 | vsphere_csi_driver_image_tag                | TRUE     | string  |                            | "v1.0.2"                  | CSI driver image tag to use                                    |
+vsphere_csi_resizer_tag                | TRUE     | string  |                            |  "v1.0.0"                  | CSI resizer image tag to use
 
 ## Usage example
 

inventory/sample/group_vars/all/vsphere.yml

@@ -7,6 +7,9 @@
 # external_vsphere_datacenter: "DATACENTER_name"
 # external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
 
+## Vsphere version where located VMs
+# external_vsphere_version: "6.7u3"
+
 ## Tags for the external vSphere Cloud Provider images
 # external_vsphere_cloud_controller_image_tag: "latest"
 # vsphere_syncer_image_tag: "v1.0.2"
@@ -14,6 +17,7 @@
 # vsphere_csi_controller: "v1.0.2"
 # vsphere_csi_liveness_probe_image_tag: "v1.1.0"
 # vsphere_csi_provisioner_image_tag: "v1.2.2"
+# vsphere_csi_resizer_tag: "v1.0.0"
 
 ## To use vSphere CSI plugin to provision volumes set this value to true
 # vsphere_csi_enabled: true

roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml

@@ -2,6 +2,7 @@
 external_vsphere_vcenter_port: "443"
 external_vsphere_insecure: "true"
 external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
+external_vsphere_version: "6.7u3"
 
 vsphere_syncer_image_tag: "v1.0.2"
 vsphere_csi_attacher_image_tag: "v1.1.1"
@@ -10,5 +11,8 @@ vsphere_csi_liveness_probe_image_tag: "v1.1.0"
 vsphere_csi_provisioner_image_tag: "v1.2.2"
 vsphere_csi_node_driver_registrar_image_tag: "v1.1.0"
 vsphere_csi_driver_image_tag: "v1.0.2"
+vsphere_csi_resizer_tag: "v1.0.0"
 
 vsphere_csi_controller_replicas: 1
+
+csi_endpoint: '{% if external_vsphere_version >= "7.0u1" %}/csi{% else %}/var/lib/csi/sockets/pluginproxy{% endif %}'

roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml

@@ -2,6 +2,11 @@
 - include_tasks: vsphere-credentials-check.yml
   tags: vsphere-csi-driver
 
+- name: vSphere CSI Driver | Choose how to deploy CSI driver based on controller version
+  set_fact:
+    controller_spec: "{% if vsphere_csi_controller is version('v2.0.0', '<') %}vsphere-csi-controller-ss.yml{% else %}vsphere-csi-controller-deployment.yml{% endif %}"
+  tags: vsphere-csi-driver
+
 - name: vSphere CSI Driver | Generate CSI cloud-config
   template:
     src: "{{ item }}.j2"
@@ -18,7 +23,7 @@
     dest: "{{ kube_config_dir }}/{{ item }}"
   with_items:
     - vsphere-csi-controller-rbac.yml
-    - vsphere-csi-controller-ss.yml
+    - "{{ controller_spec }}"
     - vsphere-csi-node.yml
   register: vsphere_csi_manifests
   when: inventory_hostname == groups['kube_control_plane'][0]

roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2

@@ -0,0 +1,188 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: vsphere-csi-controller
+  namespace: kube-system
+spec:
+  replicas: {{ vsphere_csi_controller_replicas }}
+  selector:
+    matchLabels:
+      app: vsphere-csi-controller
+  template:
+    metadata:
+      labels:
+        app: vsphere-csi-controller
+        role: vsphere-csi
+    spec:
+      serviceAccountName: vsphere-csi-controller
+      nodeSelector:
+{% if kube_version is version('v1.20.0', '<') %}
+        node-role.kubernetes.io/master: ""
+{% else %}
+        node-role.kubernetes.io/control-plane: ""
+{% endif %}
+      tolerations:
+        - operator: "Exists"
+          key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        - operator: "Exists"
+          key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+      dnsPolicy: "Default"
+      containers:
+        - name: csi-attacher
+          image: {{ quay_image_repo }}/k8scsi/csi-attacher:{{ vsphere_csi_attacher_image_tag }}
+          args:
+            - "--v=4"
+            - "--timeout=300s"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+{% if external_vsphere_version >= "7.0" %}
+        - name: csi-resizer
+          image: {{ gcr_image_repo }}/k8scsi/csi-resizer:{{ vsphere_csi_resizer_tag }}
+          args:
+            - "--v=4"
+            - "--timeout=300s"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+{% endif %}
+        - name: vsphere-csi-controller
+          image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_controller }}
+{% if external_vsphere_version >= "7.0u1" %}
+          args:
+            - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
+            - "--fss-namespace=$(CSI_NAMESPACE)"
+{% endif %}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://{{ csi_endpoint }}/csi.sock
+            - name: X_CSI_MODE
+              value: "controller"
+            - name: VSPHERE_CSI_CONFIG
+              value: "/etc/cloud/csi-vsphere.conf"
+            - name: LOGGER_LEVEL
+              value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
+{% if external_vsphere_version >= "7.0u1" %}
+            - name: INCLUSTER_CLIENT_QPS
+              value: "100"
+            - name: INCLUSTER_CLIENT_BURST
+              value: "100"
+            - name: CSI_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+{% endif %}
+            - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT
+              value: 3m
+          volumeMounts:
+            - mountPath: /etc/cloud
+              name: vsphere-config-volume
+              readOnly: true
+            - mountPath: {{ csi_endpoint }}
+              name: socket-dir
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 5
+            failureThreshold: 3
+        - name: liveness-probe
+          image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
+          args:
+            - "--v=4"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: {{ csi_endpoint }}/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: {{ csi_endpoint }}
+        - name: vsphere-syncer
+          image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/syncer:{{ vsphere_syncer_image_tag }}
+          args:
+            - "--leader-election"
+{% if external_vsphere_version >= "7.0u1" %}
+            - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
+            - "--fss-namespace=$(CSI_NAMESPACE)"
+{% endif %}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: FULL_SYNC_INTERVAL_MINUTES
+              value: "30"
+            - name: VSPHERE_CSI_CONFIG
+              value: "/etc/cloud/csi-vsphere.conf"
+            - name: LOGGER_LEVEL
+              value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
+{% if external_vsphere_version >= "7.0u1" %}
+            - name: INCLUSTER_CLIENT_QPS
+              value: "100"
+            - name: INCLUSTER_CLIENT_BURST
+              value: "100"
+            - name: CSI_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+{% endif %}
+          volumeMounts:
+            - mountPath: /etc/cloud
+              name: vsphere-config-volume
+              readOnly: true
+        - name: csi-provisioner
+          image: {{ quay_image_repo }}/k8scsi/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }}
+          args:
+            - "--v=4"
+            - "--timeout=300s"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election"
+            - "--default-fstype=ext4"
+            # needed only for topology aware setup
+            #- "--feature-gates=Topology=true"
+            #- "--strict-topology"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+      volumes:
+      - name: vsphere-config-volume
+        secret:
+          secretName: vsphere-config-secret
+      - name: socket-dir
+        emptyDir: {}
+---
+apiVersion: v1
+data:
+  "csi-migration": "false" # csi-migration feature is only available for vSphere 7.0U1
+kind: ConfigMap
+metadata:
+  name: internal-feature-states.csi.vsphere.vmware.com
+  namespace: kube-system
+---
+apiVersion: storage.k8s.io/v1 # For k8s 1.17 or lower use storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  name: csi.vsphere.vmware.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false
+

roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2

@@ -10,23 +10,47 @@ metadata:
   name: vsphere-csi-controller-role
 rules:
   - apiGroups: [""]
-    resources: ["nodes", "persistentvolumeclaims", "pods"]
+    resources: ["nodes", "persistentvolumeclaims", "pods", "configmaps"]
     verbs: ["get", "list", "watch"]
+{% if external_vsphere_version >= "7.0" %}
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+{% if external_vsphere_version >= "7.0u1" %}
+    verbs: ["patch"]
+{% else %}
+    verbs: ["update", "patch"]
+{% endif %}
+{% endif %}
   - apiGroups: [""]
     resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "update", "delete"]
+    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["get", "list", "watch", "create", "update", "patch"]
+{% if vsphere_csi_controller is version('v2.0.0', '>=') %}
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+{% endif %}
   - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["csinodes"]
+    resources: ["storageclasses","csinodes"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "patch"]
+{% if external_vsphere_version >= "7.0u1" %}
+  - apiGroups: ["cns.vmware.com"]
+    resources: ["cnsvspherevolumemigrations"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "create"]
+{% endif %}
+{% if vsphere_csi_controller is version('v2.0.0', '>=') %}
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+{% endif %}
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2

@@ -9,6 +9,8 @@ spec:
       app: vsphere-csi-node
   updateStrategy:
     type: "RollingUpdate"
+    rollingUpdate:
+      maxUnavailable: 1
   template:
     metadata:
       labels:
@@ -17,105 +19,139 @@ spec:
     spec:
       dnsPolicy: "Default"
       containers:
-        - name: node-driver-registrar
-          image: {{ quay_image_repo }}/k8scsi/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }}
-          lifecycle:
-            preStop:
-              exec:
-                command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com-reg.sock"]
-          args:
-            - "--v=5"
-            - "--csi-address=$(ADDRESS)"
-            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-            - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com/csi.sock
-          securityContext:
-            privileged: true
-          volumeMounts:
-            - name: plugin-dir
-              mountPath: /csi
-            - name: registration-dir
-              mountPath: /registration
-        - name: vsphere-csi-node
-          image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }}
-          imagePullPolicy: {{ k8s_image_pull_policy }}
-          env:
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi.sock
-            - name: X_CSI_MODE
-              value: "node"
-            - name: X_CSI_SPEC_REQ_VALIDATION
-              value: "false"
-            # needed only for topology aware setups
-            #- name: VSPHERE_CSI_CONFIG
-            #  value: "/etc/cloud/csi-vsphere.conf" # here csi-vsphere.conf is the name of the file used for creating secret using "--from-file" flag
-          args:
-            - "--v=4"
-          securityContext:
-            privileged: true
-            capabilities:
-              add: ["SYS_ADMIN"]
-            allowPrivilegeEscalation: true
-          volumeMounts:
-            # needed only for topology aware setups
-            #- name: vsphere-config-volume
-            #  mountPath: /etc/cloud
-            #  readOnly: true
-            - name: plugin-dir
-              mountPath: /csi
-            - name: pods-mount-dir
-              mountPath: /var/lib/kubelet
-              # needed so that any mounts setup inside this container are
-              # propagated back to the host machine.
-              mountPropagation: "Bidirectional"
-            - name: device-dir
-              mountPath: /dev
-          ports:
-            - name: healthz
-              containerPort: 9808
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: healthz
-            initialDelaySeconds: 10
-            timeoutSeconds: 3
-            periodSeconds: 5
-            failureThreshold: 3
-        - name: liveness-probe
-          image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
-          args:
-            - "--csi-address=$(ADDRESS)"
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-          volumeMounts:
-            - name: plugin-dir
-              mountPath: /csi
-      volumes:
+      - name: node-driver-registrar
+        image: {{ quay_image_repo }}/k8scsi/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }}
+{% if external_vsphere_version < "7.0u1" %}
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock"]
+{% endif %}
+        args:
+        - "--v=5"
+        - "--csi-address=$(ADDRESS)"
+        - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
+{% if external_vsphere_version >= "7.0u1" %}
+        - "--health-port=9809"
+{% endif %}
+        env:
+        - name: ADDRESS
+          value: /csi/csi.sock
+        - name: DRIVER_REG_SOCK_PATH
+          value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+        - name: registration-dir
+          mountPath: /registration
+{% if external_vsphere_version >= "7.0u1" %}
+        ports:
+        - containerPort: 9809
+          name: healthz
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+{% endif %}
+      - name: vsphere-csi-node
+        image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+{% if external_vsphere_version >= "7.0u1" %}
+        args:
+          - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
+          - "--fss-namespace=$(CSI_NAMESPACE)"
+{% endif %}
+        imagePullPolicy: "Always"
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: CSI_ENDPOINT
+          value: unix:///csi/csi.sock
+        - name: X_CSI_MODE
+          value: "node"
+        - name: X_CSI_SPEC_REQ_VALIDATION
+          value: "false"
+        # needed only for topology aware setups
+        #- name: VSPHERE_CSI_CONFIG
+        #  value: "/etc/cloud/csi-vsphere.conf" # here csi-vsphere.conf is the name of the file used for creating secret using "--from-file" flag
+        - name: X_CSI_DEBUG
+          value: "true"
+        - name: LOGGER_LEVEL
+          value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
+{% if external_vsphere_version >= "7.0u1" %}
+        - name: CSI_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+{% endif %}
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+          allowPrivilegeEscalation: true
+        volumeMounts:
         # needed only for topology aware setups
         #- name: vsphere-config-volume
-        #  secret:
-        #    secretName: vsphere-config-secret
-        - name: registration-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins_registry
-            type: DirectoryOrCreate
+        #  mountPath: /etc/cloud
+        #  readOnly: true
         - name: plugin-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com
-            type: DirectoryOrCreate
+          mountPath: /csi
         - name: pods-mount-dir
-          hostPath:
-            path: /var/lib/kubelet
-            type: Directory
+          mountPath: /var/lib/kubelet
+          # needed so that any mounts setup inside this container are
+          # propagated back to the host machine.
+          mountPropagation: "Bidirectional"
         - name: device-dir
-          hostPath:
-            path: /dev
+          mountPath: /dev
+        ports:
+          - containerPort: 9808
+            name: healthz
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          timeoutSeconds: 5
+          periodSeconds: 5
+          failureThreshold: 3
+      - name: liveness-probe
+        image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
+        args:
+{% if external_vsphere_version >= "7.0u1" %}
+          - "--v=4"
+{% endif %}
+          - "--csi-address=/csi/csi.sock"
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+      volumes:
+      # needed only for topology aware setups
+      #- name: vsphere-config-volume
+      #  secret:
+      #    secretName: vsphere-config-secret
+      - name: registration-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins_registry
+          type: Directory
+      - name: plugin-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com
+          type: DirectoryOrCreate
+      - name: pods-mount-dir
+        hostPath:
+          path: /var/lib/kubelet
+          type: Directory
+      - name: device-dir
+        hostPath:
+          path: /dev
+      tolerations:
+        - effect: NoExecute
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists