Add snapshot-controller for CSI drivers and snapshot CRDs, add a default volumesnapshotclass when running cinder CSI (#6537)

* add snapshot-controller and v1beta1 snapshot api

* fix typo

* udpate manifest to v1beta1

* update

* update manifests

* fix spelling

* wait until crd is applied

* fix missing info in kube module

* revert snapshotclass

* add snapshot crds before applying the csi driver

* add crds, missed them in last commit

* use pull policy from kubespray

library/kube.py

@@ -51,6 +51,11 @@ options:
     default: false
     description:
       - A flag to indicate to force delete, replace, or stop.
+  wait:
+    required: false
+    default: false
+    description:
+      - A flag to indicate to wait for resources to be created before continuing to the next step
   all:
     required: false
     default: false
@@ -130,6 +135,7 @@ class KubeManager(object):
 
         self.all = module.params.get('all')
         self.force = module.params.get('force')
+        self.wait = module.params.get('wait')
         self.name = module.params.get('name')
         self.filename = [f.strip() for f in module.params.get('filename') or []]
         self.resource = module.params.get('resource')
@@ -164,6 +170,9 @@ class KubeManager(object):
         if force:
             cmd.append('--force')
 
+        if self.wait:
+            cmd.append('--wait')
+
         if self.recursive:
             cmd.append('--recursive={}'.format(self.recursive))
 
@@ -181,6 +190,9 @@ class KubeManager(object):
         if force:
             cmd.append('--force')
 
+        if self.wait:
+            cmd.append('--wait')
+
         if self.recursive:
             cmd.append('--recursive={}'.format(self.recursive))
 
@@ -299,6 +311,7 @@ def main():
             server=dict(),
             kubectl=dict(),
             force=dict(default=False, type='bool'),
+            wait=dict(default=False, type='bool'),
             all=dict(default=False, type='bool'),
             log_level=dict(default=0, type='int'),
             state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped']),

roles/download/defaults/main.yml

@@ -597,15 +597,17 @@ addon_resizer_image_tag: "{{ addon_resizer_version }}"
 csi_attacher_image_repo: "{{ quay_image_repo }}/k8scsi/csi-attacher"
 csi_attacher_image_tag: "v2.2.0"
 csi_provisioner_image_repo: "{{ quay_image_repo }}/k8scsi/csi-provisioner"
-csi_provisioner_image_tag: "v1.5.0"
+csi_provisioner_image_tag: "v1.6.0"
 csi_snapshotter_image_repo: "{{ quay_image_repo }}/k8scsi/csi-snapshotter"
 csi_snapshotter_image_tag: "v2.1.1"
 csi_resizer_image_repo: "{{ quay_image_repo }}/k8scsi/csi-resizer"
 csi_resizer_image_tag: "v0.5.0"
 csi_node_driver_registrar_image_repo: "{{ quay_image_repo }}/k8scsi/csi-node-driver-registrar"
-csi_node_driver_registrar_image_tag: "v1.2.0"
+csi_node_driver_registrar_image_tag: "v1.3.0"
 csi_livenessprobe_image_repo: "{{ quay_image_repo }}/k8scsi/livenessprobe"
 csi_livenessprobe_image_tag: "v2.0.0"
+snapshot_controller_image_repo: "{{ quay_image_repo }}/k8scsi/snapshot-controller"
+snapshot_controller_image_tag: "v2.0.1"
 
 cinder_csi_plugin_image_repo: "{{ docker_image_repo }}/k8scloudprovider/cinder-csi-plugin"
 cinder_csi_plugin_image_tag: "v1.18.0"
@@ -1164,6 +1166,15 @@ downloads:
     groups:
     - kube-node
 
+  snapshot_controller:
+    enabled: "{{ cinder_csi_enabled }}"
+    container: true
+    repo: "{{ snapshot_controller_image_repo }}"
+    tag: "{{ snapshot_controller_image_tag }}"
+    sha256: "{{ snapshot_controller_digest_checksum|default(None) }}"
+    groups:
+    - kube-node
+
   csi_resizer:
     enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
     container: true

roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2

@@ -8,7 +8,7 @@ metadata:
   namespace: kube-system
 
 ---
-# external attacher 
+# external attacher
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -122,6 +122,9 @@ rules:
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots/status"]
     verbs: ["update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
   - apiGroups: ["apiextensions.k8s.io"]
     resources: ["customresourcedefinitions"]
     verbs: ["create", "list", "watch", "delete"]
@@ -206,4 +209,4 @@ subjects:
 roleRef:
   kind: Role
   name: external-resizer-cfg
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2

@@ -14,7 +14,15 @@ rules:
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["get", "list", "watch", "create", "update", "patch"]
-
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: CSI CRD | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: volumesnapshotclasses, file: volumesnapshotclasses.yml}
+    - {name: volumesnapshotcontents, file: volumesnapshotcontents.yml}
+    - {name: volumesnapshots, file: volumesnapshots.yml}
+  register: csi_crd_manifests
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: csi-driver
+
+- name: CSI CRD | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+    wait: true
+  with_items:
+    - "{{ csi_crd_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+  tags: csi-driver

roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2

@@ -0,0 +1,84 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260"
+  creationTimestamp: null
+  name: volumesnapshotclasses.snapshot.storage.k8s.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .driver
+    name: Driver
+    type: string
+  - JSONPath: .deletionPolicy
+    description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass
+      should be deleted when its bound VolumeSnapshot is deleted.
+    name: DeletionPolicy
+    type: string
+  - JSONPath: .metadata.creationTimestamp
+    name: Age
+    type: date
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotClass
+    listKind: VolumeSnapshotClassList
+    plural: volumesnapshotclasses
+    singular: volumesnapshotclass
+  preserveUnknownFields: false
+  scope: Cluster
+  subresources: {}
+  validation:
+    openAPIV3Schema:
+      description: VolumeSnapshotClass specifies parameters that a underlying storage
+        system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+        is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+        are non-namespaced
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        deletionPolicy:
+          description: deletionPolicy determines whether a VolumeSnapshotContent created
+            through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted. Supported values are "Retain" and "Delete". "Retain" means
+            that the VolumeSnapshotContent and its physical snapshot on underlying
+            storage system are kept. "Delete" means that the VolumeSnapshotContent
+            and its physical snapshot on underlying storage system are deleted. Required.
+          enum:
+          - Delete
+          - Retain
+          type: string
+        driver:
+          description: driver is the name of the storage driver that handles this
+            VolumeSnapshotClass. Required.
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        parameters:
+          additionalProperties:
+            type: string
+          description: parameters is a key-value map with storage driver specific
+            parameters for creating snapshots. These values are opaque to Kubernetes.
+          type: object
+      required:
+      - deletionPolicy
+      - driver
+      type: object
+  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2

@@ -0,0 +1,232 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260"
+  creationTimestamp: null
+  name: volumesnapshotcontents.snapshot.storage.k8s.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .status.readyToUse
+    description: Indicates if a snapshot is ready to be used to restore a volume.
+    name: ReadyToUse
+    type: boolean
+  - JSONPath: .status.restoreSize
+    description: Represents the complete size of the snapshot in bytes
+    name: RestoreSize
+    type: integer
+  - JSONPath: .spec.deletionPolicy
+    description: Determines whether this VolumeSnapshotContent and its physical snapshot
+      on the underlying storage system should be deleted when its bound VolumeSnapshot
+      is deleted.
+    name: DeletionPolicy
+    type: string
+  - JSONPath: .spec.driver
+    description: Name of the CSI driver used to create the physical snapshot on the
+      underlying storage system.
+    name: Driver
+    type: string
+  - JSONPath: .spec.volumeSnapshotClassName
+    description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+    name: VolumeSnapshotClass
+    type: string
+  - JSONPath: .spec.volumeSnapshotRef.name
+    description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+      object is bound.
+    name: VolumeSnapshot
+    type: string
+  - JSONPath: .metadata.creationTimestamp
+    name: Age
+    type: date
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotContent
+    listKind: VolumeSnapshotContentList
+    plural: volumesnapshotcontents
+    singular: volumesnapshotcontent
+  preserveUnknownFields: false
+  scope: Cluster
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+        object in the underlying storage system
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        spec:
+          description: spec defines properties of a VolumeSnapshotContent created
+            by the underlying storage system. Required.
+          properties:
+            deletionPolicy:
+              description: deletionPolicy determines whether this VolumeSnapshotContent
+                and its physical snapshot on the underlying storage system should
+                be deleted when its bound VolumeSnapshot is deleted. Supported values
+                are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                and its physical snapshot on underlying storage system are kept. "Delete"
+                means that the VolumeSnapshotContent and its physical snapshot on
+                underlying storage system are deleted. In dynamic snapshot creation
+                case, this field will be filled in with the "DeletionPolicy" field
+                defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For
+                pre-existing snapshots, users MUST specify this field when creating
+                the VolumeSnapshotContent object. Required.
+              enum:
+              - Delete
+              - Retain
+              type: string
+            driver:
+              description: driver is the name of the CSI driver used to create the
+                physical snapshot on the underlying storage system. This MUST be the
+                same as the name returned by the CSI GetPluginName() call for that
+                driver. Required.
+              type: string
+            source:
+              description: source specifies from where a snapshot will be created.
+                This field is immutable after creation. Required.
+              properties:
+                snapshotHandle:
+                  description: snapshotHandle specifies the CSI "snapshot_id" of a
+                    pre-existing snapshot on the underlying storage system. This field
+                    is immutable.
+                  type: string
+                volumeHandle:
+                  description: volumeHandle specifies the CSI "volume_id" of the volume
+                    from which a snapshot should be dynamically taken from. This field
+                    is immutable.
+                  type: string
+              type: object
+            volumeSnapshotClassName:
+              description: name of the VolumeSnapshotClass to which this snapshot
+                belongs.
+              type: string
+            volumeSnapshotRef:
+              description: volumeSnapshotRef specifies the VolumeSnapshot object to
+                which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                field must reference to this VolumeSnapshotContent's name for the
+                bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                object, name and namespace of the VolumeSnapshot object MUST be provided
+                for binding to happen. This field is immutable after creation. Required.
+              properties:
+                apiVersion:
+                  description: API version of the referent.
+                  type: string
+                fieldPath:
+                  description: 'If referring to a piece of an object instead of an
+                    entire object, this string should contain a valid JSON/Go field
+                    access statement, such as desiredState.manifest.containers[2].
+                    For example, if the object reference is to a container within
+                    a pod, this would take on a value like: "spec.containers{name}"
+                    (where "name" refers to the name of the container that triggered
+                    the event) or if no container name is specified "spec.containers[2]"
+                    (container with index 2 in this pod). This syntax is chosen only
+                    to have some well-defined way of referencing a part of an object.
+                    TODO: this design is not final and this field is subject to change
+                    in the future.'
+                  type: string
+                kind:
+                  description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                  type: string
+                name:
+                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                  type: string
+                namespace:
+                  description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                  type: string
+                resourceVersion:
+                  description: 'Specific resourceVersion to which this reference is
+                    made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                  type: string
+                uid:
+                  description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                  type: string
+              type: object
+          required:
+          - deletionPolicy
+          - driver
+          - source
+          - volumeSnapshotRef
+          type: object
+        status:
+          description: status represents the current information of a snapshot.
+          properties:
+            creationTime:
+              description: creationTime is the timestamp when the point-in-time snapshot
+                is taken by the underlying storage system. In dynamic snapshot creation
+                case, this field will be filled in with the "creation_time" value
+                returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing
+                snapshot, this field will be filled with the "creation_time" value
+                returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                it. If not specified, it indicates the creation time is unknown. The
+                format of this field is a Unix nanoseconds time encoded as an int64.
+                On Unix, the command `date +%s%N` returns the current time in nanoseconds
+                since 1970-01-01 00:00:00 UTC.
+              format: int64
+              type: integer
+            error:
+              description: error is the latest observed error during snapshot creation,
+                if any.
+              properties:
+                message:
+                  description: 'message is a string detailing the encountered error
+                    during snapshot creation if specified. NOTE: message may be logged,
+                    and it should not contain sensitive information.'
+                  type: string
+                time:
+                  description: time is the timestamp when the error was encountered.
+                  format: date-time
+                  type: string
+              type: object
+            readyToUse:
+              description: readyToUse indicates if a snapshot is ready to be used
+                to restore a volume. In dynamic snapshot creation case, this field
+                will be filled in with the "ready_to_use" value returned from CSI
+                "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this
+                field will be filled with the "ready_to_use" value returned from the
+                CSI "ListSnapshots" gRPC call if the driver supports it, otherwise,
+                this field will be set to "True". If not specified, it means the readiness
+                of a snapshot is unknown.
+              type: boolean
+            restoreSize:
+              description: restoreSize represents the complete size of the snapshot
+                in bytes. In dynamic snapshot creation case, this field will be filled
+                in with the "size_bytes" value returned from CSI "CreateSnapshotRequest"
+                gRPC call. For a pre-existing snapshot, this field will be filled
+                with the "size_bytes" value returned from the CSI "ListSnapshots"
+                gRPC call if the driver supports it. When restoring a volume from
+                this snapshot, the size of the volume MUST NOT be smaller than the
+                restoreSize if it is specified, otherwise the restoration will fail.
+                If not specified, it indicates that the size is unknown.
+              format: int64
+              minimum: 0
+              type: integer
+            snapshotHandle:
+              description: snapshotHandle is the CSI "snapshot_id" of a snapshot on
+                the underlying storage system. If not specified, it indicates that
+                dynamic snapshot creation has either failed or it is still in progress.
+              type: string
+          type: object
+      required:
+      - spec
+      type: object
+  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2

@@ -0,0 +1,187 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260"
+  creationTimestamp: null
+  name: volumesnapshots.snapshot.storage.k8s.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .status.readyToUse
+    description: Indicates if a snapshot is ready to be used to restore a volume.
+    name: ReadyToUse
+    type: boolean
+  - JSONPath: .spec.source.persistentVolumeClaimName
+    description: Name of the source PVC from where a dynamically taken snapshot will
+      be created.
+    name: SourcePVC
+    type: string
+  - JSONPath: .spec.source.volumeSnapshotContentName
+    description: Name of the VolumeSnapshotContent which represents a pre-provisioned
+      snapshot.
+    name: SourceSnapshotContent
+    type: string
+  - JSONPath: .status.restoreSize
+    description: Represents the complete size of the snapshot.
+    name: RestoreSize
+    type: string
+  - JSONPath: .spec.volumeSnapshotClassName
+    description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+    name: SnapshotClass
+    type: string
+  - JSONPath: .status.boundVolumeSnapshotContentName
+    description: The name of the VolumeSnapshotContent to which this VolumeSnapshot
+      is bound.
+    name: SnapshotContent
+    type: string
+  - JSONPath: .status.creationTime
+    description: Timestamp when the point-in-time snapshot is taken by the underlying
+      storage system.
+    name: CreationTime
+    type: date
+  - JSONPath: .metadata.creationTimestamp
+    name: Age
+    type: date
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshot
+    listKind: VolumeSnapshotList
+    plural: volumesnapshots
+    singular: volumesnapshot
+  preserveUnknownFields: false
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: VolumeSnapshot is a user's request for either creating a point-in-time
+        snapshot of a persistent volume, or binding to a pre-existing snapshot.
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        spec:
+          description: 'spec defines the desired characteristics of a snapshot requested
+            by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+            Required.'
+          properties:
+            source:
+              description: source specifies where a snapshot will be created from.
+                This field is immutable after creation. Required.
+              properties:
+                persistentVolumeClaimName:
+                  description: persistentVolumeClaimName specifies the name of the
+                    PersistentVolumeClaim object in the same namespace as the VolumeSnapshot
+                    object where the snapshot should be dynamically taken from. This
+                    field is immutable.
+                  type: string
+                volumeSnapshotContentName:
+                  description: volumeSnapshotContentName specifies the name of a pre-existing
+                    VolumeSnapshotContent object. This field is immutable.
+                  type: string
+              type: object
+            volumeSnapshotClassName:
+              description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass
+                requested by the VolumeSnapshot. If not specified, the default snapshot
+                class will be used if one exists. If not specified, and there is no
+                default snapshot class, dynamic snapshot creation will fail. Empty
+                string is not allowed for this field. TODO(xiangqian): a webhook validation
+                on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes'
+              type: string
+          required:
+          - source
+          type: object
+        status:
+          description: 'status represents the current information of a snapshot. NOTE:
+            status can be modified by sources other than system controllers, and must
+            not be depended upon for accuracy. Controllers should only use information
+            from the VolumeSnapshotContent object after verifying that the binding
+            is accurate and complete.'
+          properties:
+            boundVolumeSnapshotContentName:
+              description: 'boundVolumeSnapshotContentName represents the name of
+                the VolumeSnapshotContent object to which the VolumeSnapshot object
+                is bound. If not specified, it indicates that the VolumeSnapshot object
+                has not been successfully bound to a VolumeSnapshotContent object
+                yet. NOTE: Specified boundVolumeSnapshotContentName alone does not
+                mean binding       is valid. Controllers MUST always verify bidirectional
+                binding between       VolumeSnapshot and VolumeSnapshotContent to
+                avoid possible security issues.'
+              type: string
+            creationTime:
+              description: creationTime is the timestamp when the point-in-time snapshot
+                is taken by the underlying storage system. In dynamic snapshot creation
+                case, this field will be filled in with the "creation_time" value
+                returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing
+                snapshot, this field will be filled with the "creation_time" value
+                returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                it. If not specified, it indicates that the creation time of the snapshot
+                is unknown.
+              format: date-time
+              type: string
+            error:
+              description: error is the last observed error during snapshot creation,
+                if any. This field could be helpful to upper level controllers(i.e.,
+                application controller) to decide whether they should continue on
+                waiting for the snapshot to be created based on the type of error
+                reported.
+              properties:
+                message:
+                  description: 'message is a string detailing the encountered error
+                    during snapshot creation if specified. NOTE: message may be logged,
+                    and it should not contain sensitive information.'
+                  type: string
+                time:
+                  description: time is the timestamp when the error was encountered.
+                  format: date-time
+                  type: string
+              type: object
+            readyToUse:
+              description: readyToUse indicates if a snapshot is ready to be used
+                to restore a volume. In dynamic snapshot creation case, this field
+                will be filled in with the "ready_to_use" value returned from CSI
+                "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this
+                field will be filled with the "ready_to_use" value returned from the
+                CSI "ListSnapshots" gRPC call if the driver supports it, otherwise,
+                this field will be set to "True". If not specified, it means the readiness
+                of a snapshot is unknown.
+              type: boolean
+            restoreSize:
+              anyOf:
+              - type: integer
+              - type: string
+              description: restoreSize represents the complete size of the snapshot
+                in bytes. In dynamic snapshot creation case, this field will be filled
+                in with the "size_bytes" value returned from CSI "CreateSnapshotRequest"
+                gRPC call. For a pre-existing snapshot, this field will be filled
+                with the "size_bytes" value returned from the CSI "ListSnapshots"
+                gRPC call if the driver supports it. When restoring a volume from
+                this snapshot, the size of the volume MUST NOT be smaller than the
+                restoreSize if it is specified, otherwise the restoration will fail.
+                If not specified, it indicates that the size is unknown.
+              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+              x-kubernetes-int-or-string: true
+          type: object
+      required:
+      - spec
+      type: object
+  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

roles/kubernetes-apps/meta/main.yml

@@ -29,6 +29,14 @@ dependencies:
       - apps
       - metrics_server
 
+  - role: kubernetes-apps/csi_driver/csi_crd
+    when:
+      - cinder_csi_enabled
+      - inventory_hostname == groups['kube-master'][0]
+    tags:
+      - apps
+      - csi-driver
+
   - role: kubernetes-apps/csi_driver/cinder
     when:
       - cinder_csi_enabled
@@ -77,6 +85,13 @@ dependencies:
       - apps
       - persistent_volumes
 
+  - role: kubernetes-apps/snapshots
+    when: inventory_hostname == groups['kube-master'][0]
+    tags:
+      - apps
+      - snapshots
+      - csi-driver
+
   - role: kubernetes-apps/container_runtimes
     when:
       - inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+snapshot_classes:
+  - name: cinder-csi-snapshot
+    is_default: false
+    force_create: true

roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+- name: Kubernetes Snapshots | Copy Cinder CSI Snapshot Class template
+  template:
+    src: "cinder-csi-snapshot-class.yml.j2"
+    dest: "{{ kube_config_dir }}/cinder-csi-snapshot-class.yml"
+  register: manifests
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Kubernetes Snapshots | Add Cinder CSI Snapshot Class
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/cinder-csi-snapshot-class.yml"
+    state: "latest"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - manifests.changed

roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2

@@ -0,0 +1,13 @@
+{% for class in snapshot_classes %}
+---
+kind: VolumeSnapshotClass
+apiVersion: snapshot.storage.k8s.io/v1beta1
+metadata:
+  name: "{{ class.name }}"
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}"
+driver: cinder.csi.openstack.org
+deletionPolicy: Delete
+parameters:
+  force-create: "{{ class.force_create }}"
+{% endfor %}

roles/kubernetes-apps/snapshots/meta/main.yml

@@ -0,0 +1,14 @@
+---
+dependencies:
+  - role: kubernetes-apps/snapshots/snapshot-controller
+    when:
+      - cinder_csi_enabled
+    tags:
+      - snapshot-controller
+
+  - role: kubernetes-apps/snapshots/cinder-csi
+    when:
+      - cinder_csi_enabled
+    tags:
+      - snapshot
+      - cinder-csi-driver

roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+snapshot_controller_replicas: 1

roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml

@@ -0,0 +1,25 @@
+---
+- name: Snapshot Controller | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: rbac-snapshot-controller, file: rbac-snapshot-controller.yml}
+    - {name: snapshot-controller, file: snapshot-controller.yml}
+  register: snapshot_controller_manifests
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: snapshot-controller
+
+- name: Snapshot Controller | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ snapshot_controller_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+  tags: snapshot-controller

roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2

@@ -0,0 +1,85 @@
+# RBAC file for the snapshot controller.
+#
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: snapshot-controller
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  # rename if there are conflicts
+  name: snapshot-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  # change the name also here if the ClusterRole gets renamed
+  name: snapshot-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: kube-system
+  name: snapshot-controller-leaderelection
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: snapshot-controller-leaderelection
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2

@@ -0,0 +1,32 @@
+# This YAML file shows how to deploy the snapshot controller
+
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: snapshot-controller
+  namespace: kube-system
+spec:
+  serviceName: "snapshot-controller"
+  replicas: {{ snapshot_controller_replicas }}
+  selector:
+    matchLabels:
+      app: snapshot-controller
+  template:
+    metadata:
+      labels:
+        app: snapshot-controller
+    spec:
+      serviceAccount: snapshot-controller
+      containers:
+        - name: snapshot-controller
+          image: {{ snapshot_controller_image_repo }}:{{ snapshot_controller_image_tag }}
+          args:
+            - "--v=5"
+            - "--leader-election=false"
+          imagePullPolicy: {{ k8s_image_pull_policy }}