From 2ff7ab8d4001418e31fcae1c1afc0647c86355ad Mon Sep 17 00:00:00 2001 From: Hugo Blom Date: Thu, 3 Sep 2020 13:01:43 +0200 Subject: [PATCH] Add snapshot-controller for CSI drivers and snapshot CRDs, add a default volumesnapshotclass when running cinder CSI (#6537) * add snapshot-controller and v1beta1 snapshot api * fix typo * udpate manifest to v1beta1 * update * update manifests * fix spelling * wait until crd is applied * fix missing info in kube module * revert snapshotclass * add snapshot crds before applying the csi driver * add crds, missed them in last commit * use pull policy from kubespray --- library/kube.py | 13 + roles/download/defaults/main.yml | 15 +- .../cinder-csi-controllerplugin-rbac.yml.j2 | 7 +- .../cinder-csi-nodeplugin-rbac.yml.j2 | 10 +- .../csi_driver/csi_crd/tasks/main.yml | 27 ++ .../templates/volumesnapshotclasses.yml.j2 | 84 +++++++ .../templates/volumesnapshotcontents.yml.j2 | 232 ++++++++++++++++++ .../csi_crd/templates/volumesnapshots.yml.j2 | 187 ++++++++++++++ roles/kubernetes-apps/meta/main.yml | 15 ++ .../snapshots/cinder-csi/defaults/main.yml | 5 + .../snapshots/cinder-csi/tasks/main.yml | 17 ++ .../cinder-csi-snapshot-class.yml.j2 | 13 + roles/kubernetes-apps/snapshots/meta/main.yml | 14 ++ .../snapshot-controller/defaults/main.yml | 2 + .../snapshot-controller/tasks/main.yml | 25 ++ .../templates/rbac-snapshot-controller.yml.j2 | 85 +++++++ .../templates/snapshot-controller.yml.j2 | 32 +++ 17 files changed, 778 insertions(+), 5 deletions(-) create mode 100644 roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml create mode 100644 roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2 create mode 100644 roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml create mode 100644 roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml create mode 100644 roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2 create mode 100644 roles/kubernetes-apps/snapshots/meta/main.yml create mode 100644 roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml create mode 100644 roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml create mode 100644 roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2 create mode 100644 roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2 diff --git a/library/kube.py b/library/kube.py index 36eed6159..8fc47bbc6 100644 --- a/library/kube.py +++ b/library/kube.py @@ -51,6 +51,11 @@ options: default: false description: - A flag to indicate to force delete, replace, or stop. + wait: + required: false + default: false + description: + - A flag to indicate to wait for resources to be created before continuing to the next step all: required: false default: false @@ -130,6 +135,7 @@ class KubeManager(object): self.all = module.params.get('all') self.force = module.params.get('force') + self.wait = module.params.get('wait') self.name = module.params.get('name') self.filename = [f.strip() for f in module.params.get('filename') or []] self.resource = module.params.get('resource') @@ -164,6 +170,9 @@ class KubeManager(object): if force: cmd.append('--force') + if self.wait: + cmd.append('--wait') + if self.recursive: cmd.append('--recursive={}'.format(self.recursive)) @@ -181,6 +190,9 @@ class KubeManager(object): if force: cmd.append('--force') + if self.wait: + cmd.append('--wait') + if self.recursive: cmd.append('--recursive={}'.format(self.recursive)) @@ -299,6 +311,7 @@ def main(): server=dict(), kubectl=dict(), force=dict(default=False, type='bool'), + wait=dict(default=False, type='bool'), all=dict(default=False, type='bool'), log_level=dict(default=0, type='int'), state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped']), diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index a90af4da2..478644b17 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -597,15 +597,17 @@ addon_resizer_image_tag: "{{ addon_resizer_version }}" csi_attacher_image_repo: "{{ quay_image_repo }}/k8scsi/csi-attacher" csi_attacher_image_tag: "v2.2.0" csi_provisioner_image_repo: "{{ quay_image_repo }}/k8scsi/csi-provisioner" -csi_provisioner_image_tag: "v1.5.0" +csi_provisioner_image_tag: "v1.6.0" csi_snapshotter_image_repo: "{{ quay_image_repo }}/k8scsi/csi-snapshotter" csi_snapshotter_image_tag: "v2.1.1" csi_resizer_image_repo: "{{ quay_image_repo }}/k8scsi/csi-resizer" csi_resizer_image_tag: "v0.5.0" csi_node_driver_registrar_image_repo: "{{ quay_image_repo }}/k8scsi/csi-node-driver-registrar" -csi_node_driver_registrar_image_tag: "v1.2.0" +csi_node_driver_registrar_image_tag: "v1.3.0" csi_livenessprobe_image_repo: "{{ quay_image_repo }}/k8scsi/livenessprobe" csi_livenessprobe_image_tag: "v2.0.0" +snapshot_controller_image_repo: "{{ quay_image_repo }}/k8scsi/snapshot-controller" +snapshot_controller_image_tag: "v2.0.1" cinder_csi_plugin_image_repo: "{{ docker_image_repo }}/k8scloudprovider/cinder-csi-plugin" cinder_csi_plugin_image_tag: "v1.18.0" @@ -1164,6 +1166,15 @@ downloads: groups: - kube-node + snapshot_controller: + enabled: "{{ cinder_csi_enabled }}" + container: true + repo: "{{ snapshot_controller_image_repo }}" + tag: "{{ snapshot_controller_image_tag }}" + sha256: "{{ snapshot_controller_digest_checksum|default(None) }}" + groups: + - kube-node + csi_resizer: enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 index c0c92de0e..353e68558 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 @@ -8,7 +8,7 @@ metadata: namespace: kube-system --- -# external attacher +# external attacher kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -122,6 +122,9 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots/status"] verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create", "list", "watch", "delete"] @@ -206,4 +209,4 @@ subjects: roleRef: kind: Role name: external-resizer-cfg - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 index 912923fca..db589636d 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 @@ -14,7 +14,15 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] - + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml b/roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml new file mode 100644 index 000000000..85d637efb --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: CSI CRD | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - {name: volumesnapshotclasses, file: volumesnapshotclasses.yml} + - {name: volumesnapshotcontents, file: volumesnapshotcontents.yml} + - {name: volumesnapshots, file: volumesnapshots.yml} + register: csi_crd_manifests + when: inventory_hostname == groups['kube-master'][0] + tags: csi-driver + +- name: CSI CRD | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + wait: true + with_items: + - "{{ csi_crd_manifests.results }}" + when: + - inventory_hostname == groups['kube-master'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: csi-driver diff --git a/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2 b/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2 new file mode 100644 index 000000000..dcc9e96f4 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2 @@ -0,0 +1,84 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + additionalPrinterColumns: + - JSONPath: .driver + name: Driver + type: string + - JSONPath: .deletionPolicy + description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass + should be deleted when its bound VolumeSnapshot is deleted. + name: DeletionPolicy + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + preserveUnknownFields: false + scope: Cluster + subresources: {} + validation: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created + through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. Supported values are "Retain" and "Delete". "Retain" means + that the VolumeSnapshotContent and its physical snapshot on underlying + storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] \ No newline at end of file diff --git a/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2 b/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2 new file mode 100644 index 000000000..db1591c8e --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2 @@ -0,0 +1,232 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + additionalPrinterColumns: + - JSONPath: .status.readyToUse + description: Indicates if a snapshot is ready to be used to restore a volume. + name: ReadyToUse + type: boolean + - JSONPath: .status.restoreSize + description: Represents the complete size of the snapshot in bytes + name: RestoreSize + type: integer + - JSONPath: .spec.deletionPolicy + description: Determines whether this VolumeSnapshotContent and its physical snapshot + on the underlying storage system should be deleted when its bound VolumeSnapshot + is deleted. + name: DeletionPolicy + type: string + - JSONPath: .spec.driver + description: Name of the CSI driver used to create the physical snapshot on the + underlying storage system. + name: Driver + type: string + - JSONPath: .spec.volumeSnapshotClassName + description: Name of the VolumeSnapshotClass to which this snapshot belongs. + name: VolumeSnapshotClass + type: string + - JSONPath: .spec.volumeSnapshotRef.name + description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + name: VolumeSnapshot + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + preserveUnknownFields: false + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. "Delete" + means that the VolumeSnapshotContent and its physical snapshot on + underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For + pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be the + same as the name returned by the CSI GetPluginName() call for that + driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a + pre-existing snapshot on the underlying storage system. This field + is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume + from which a snapshot should be dynamically taken from. This field + is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to + which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be provided + for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on + the underlying storage system. If not specified, it indicates that + dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] \ No newline at end of file diff --git a/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2 b/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2 new file mode 100644 index 000000000..cc6bcac69 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2 @@ -0,0 +1,187 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + additionalPrinterColumns: + - JSONPath: .status.readyToUse + description: Indicates if a snapshot is ready to be used to restore a volume. + name: ReadyToUse + type: boolean + - JSONPath: .spec.source.persistentVolumeClaimName + description: Name of the source PVC from where a dynamically taken snapshot will + be created. + name: SourcePVC + type: string + - JSONPath: .spec.source.volumeSnapshotContentName + description: Name of the VolumeSnapshotContent which represents a pre-provisioned + snapshot. + name: SourceSnapshotContent + type: string + - JSONPath: .status.restoreSize + description: Represents the complete size of the snapshot. + name: RestoreSize + type: string + - JSONPath: .spec.volumeSnapshotClassName + description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + name: SnapshotClass + type: string + - JSONPath: .status.boundVolumeSnapshotContentName + description: The name of the VolumeSnapshotContent to which this VolumeSnapshot + is bound. + name: SnapshotContent + type: string + - JSONPath: .status.creationTime + description: Timestamp when the point-in-time snapshot is taken by the underlying + storage system. + name: CreationTime + type: date + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + preserveUnknownFields: false + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. This + field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing + VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is no + default snapshot class, dynamic snapshot creation will fail. Empty + string is not allowed for this field. TODO(xiangqian): a webhook validation + on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. NOTE: + status can be modified by sources other than system controllers, and must + not be depended upon for accuracy. Controllers should only use information + from the VolumeSnapshotContent object after verifying that the binding + is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot object + has not been successfully bound to a VolumeSnapshotContent object + yet. NOTE: Specified boundVolumeSnapshotContentName alone does not + mean binding is valid. Controllers MUST always verify bidirectional + binding between VolumeSnapshot and VolumeSnapshotContent to + avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates that the creation time of the snapshot + is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + anyOf: + - type: integer + - type: string + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] \ No newline at end of file diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 1c9d69adc..4d6d23d5c 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -29,6 +29,14 @@ dependencies: - apps - metrics_server + - role: kubernetes-apps/csi_driver/csi_crd + when: + - cinder_csi_enabled + - inventory_hostname == groups['kube-master'][0] + tags: + - apps + - csi-driver + - role: kubernetes-apps/csi_driver/cinder when: - cinder_csi_enabled @@ -77,6 +85,13 @@ dependencies: - apps - persistent_volumes + - role: kubernetes-apps/snapshots + when: inventory_hostname == groups['kube-master'][0] + tags: + - apps + - snapshots + - csi-driver + - role: kubernetes-apps/container_runtimes when: - inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml b/roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml new file mode 100644 index 000000000..7b5dd732f --- /dev/null +++ b/roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml @@ -0,0 +1,5 @@ +--- +snapshot_classes: + - name: cinder-csi-snapshot + is_default: false + force_create: true diff --git a/roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml b/roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml new file mode 100644 index 000000000..32940af08 --- /dev/null +++ b/roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: Kubernetes Snapshots | Copy Cinder CSI Snapshot Class template + template: + src: "cinder-csi-snapshot-class.yml.j2" + dest: "{{ kube_config_dir }}/cinder-csi-snapshot-class.yml" + register: manifests + when: + - inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Snapshots | Add Cinder CSI Snapshot Class + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/cinder-csi-snapshot-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube-master'][0] + - manifests.changed diff --git a/roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2 b/roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2 new file mode 100644 index 000000000..b7e649f66 --- /dev/null +++ b/roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2 @@ -0,0 +1,13 @@ +{% for class in snapshot_classes %} +--- +kind: VolumeSnapshotClass +apiVersion: snapshot.storage.k8s.io/v1beta1 +metadata: + name: "{{ class.name }}" + annotations: + storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}" +driver: cinder.csi.openstack.org +deletionPolicy: Delete +parameters: + force-create: "{{ class.force_create }}" +{% endfor %} diff --git a/roles/kubernetes-apps/snapshots/meta/main.yml b/roles/kubernetes-apps/snapshots/meta/main.yml new file mode 100644 index 000000000..9b0e92be4 --- /dev/null +++ b/roles/kubernetes-apps/snapshots/meta/main.yml @@ -0,0 +1,14 @@ +--- +dependencies: + - role: kubernetes-apps/snapshots/snapshot-controller + when: + - cinder_csi_enabled + tags: + - snapshot-controller + + - role: kubernetes-apps/snapshots/cinder-csi + when: + - cinder_csi_enabled + tags: + - snapshot + - cinder-csi-driver diff --git a/roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml b/roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml new file mode 100644 index 000000000..9c757fb60 --- /dev/null +++ b/roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml @@ -0,0 +1,2 @@ +--- +snapshot_controller_replicas: 1 diff --git a/roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml b/roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml new file mode 100644 index 000000000..feeee4a41 --- /dev/null +++ b/roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml @@ -0,0 +1,25 @@ +--- +- name: Snapshot Controller | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - {name: rbac-snapshot-controller, file: rbac-snapshot-controller.yml} + - {name: snapshot-controller, file: snapshot-controller.yml} + register: snapshot_controller_manifests + when: inventory_hostname == groups['kube-master'][0] + tags: snapshot-controller + +- name: Snapshot Controller | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ snapshot_controller_manifests.results }}" + when: + - inventory_hostname == groups['kube-master'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: snapshot-controller diff --git a/roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2 b/roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2 new file mode 100644 index 000000000..277b87b84 --- /dev/null +++ b/roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2 @@ -0,0 +1,85 @@ +# RBAC file for the snapshot controller. +# +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-controller + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # rename if there are conflicts + name: snapshot-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-role +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: kube-system +roleRef: + kind: ClusterRole + # change the name also here if the ClusterRole gets renamed + name: snapshot-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: kube-system + name: snapshot-controller-leaderelection +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: kube-system +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: kube-system +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2 b/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2 new file mode 100644 index 000000000..42e0cb455 --- /dev/null +++ b/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2 @@ -0,0 +1,32 @@ +# This YAML file shows how to deploy the snapshot controller + +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: snapshot-controller + namespace: kube-system +spec: + serviceName: "snapshot-controller" + replicas: {{ snapshot_controller_replicas }} + selector: + matchLabels: + app: snapshot-controller + template: + metadata: + labels: + app: snapshot-controller + spec: + serviceAccount: snapshot-controller + containers: + - name: snapshot-controller + image: {{ snapshot_controller_image_repo }}:{{ snapshot_controller_image_tag }} + args: + - "--v=5" + - "--leader-election=false" + imagePullPolicy: {{ k8s_image_pull_policy }}