From deef47c923c44069d154efc9f174cb3f30669594 Mon Sep 17 00:00:00 2001 From: Wong Hoi Sing Edison Date: Thu, 15 Feb 2018 09:55:43 +0800 Subject: [PATCH] Upgrade Local Volume Provisioner Addon to v2.0.0 --- cluster.yml | 1 + inventory/sample/group_vars/k8s-cluster.yml | 8 ++- .../local_volume_provisioner/README.md | 59 ++++++++++--------- .../defaults/main.yml | 8 +++ .../local_volume_provisioner/tasks/main.yml | 24 ++++---- ...ume-provisioner-clusterrolebinding.yml.j2} | 21 +++---- .../local-volume-provisioner-cm.yml.j2 | 11 ++++ .../local-volume-provisioner-ds.yml.j2 | 45 ++++++++++++++ .../local-volume-provisioner-ns.yml.j2 | 5 ++ .../local-volume-provisioner-sa.yml.j2 | 6 ++ .../local-volume-provisioner-sc.yml.j2 | 7 +++ .../external_provisioner/meta/main.yml | 8 +++ .../defaults/main.yml | 6 -- .../templates/configmap.yml.j2 | 14 ----- .../templates/daemonset.yml.j2 | 45 -------------- .../templates/serviceaccount.yml.j2 | 5 -- roles/kubernetes-apps/meta/main.yml | 7 --- .../node/templates/kubelet-container.j2 | 2 +- .../node/templates/kubelet.rkt.service.j2 | 4 +- roles/kubernetes/preinstall/tasks/main.yml | 10 +++- roles/kubespray-defaults/defaults/main.yaml | 13 ++-- 21 files changed, 170 insertions(+), 139 deletions(-) rename docs/local-storage-provisioner.md => roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md (64%) create mode 100644 roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml rename roles/kubernetes-apps/{ => external_provisioner}/local_volume_provisioner/tasks/main.yml (52%) rename roles/kubernetes-apps/{local_volume_provisioner/templates/clusterrolebinding.yml.j2 => external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2} (51%) create mode 100644 roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 create mode 100644 roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 create mode 100644 roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2 create mode 100644 roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2 create mode 100644 roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 create mode 100644 roles/kubernetes-apps/external_provisioner/meta/main.yml delete mode 100644 roles/kubernetes-apps/local_volume_provisioner/defaults/main.yml delete mode 100644 roles/kubernetes-apps/local_volume_provisioner/templates/configmap.yml.j2 delete mode 100644 roles/kubernetes-apps/local_volume_provisioner/templates/daemonset.yml.j2 delete mode 100644 roles/kubernetes-apps/local_volume_provisioner/templates/serviceaccount.yml.j2 diff --git a/cluster.yml b/cluster.yml index 995e36309..b388c9ced 100644 --- a/cluster.yml +++ b/cluster.yml @@ -94,6 +94,7 @@ - { role: kubespray-defaults} - { role: kubernetes-apps/network_plugin, tags: network } - { role: kubernetes-apps/policy_controller, tags: policy-controller } + - { role: kubernetes-apps/external_provisioner, tags: external-provisioner } - hosts: calico-rr any_errors_fatal: "{{ any_errors_fatal | default(true) }}" diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index cf4f08a89..8ea6e27ca 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -169,9 +169,11 @@ istio_enabled: false registry_enabled: false # Local volume provisioner deployment -# deprecated will be removed -local_volumes_enabled: false -local_volume_provisioner_enabled: "{{ local_volumes_enabled }}" +local_volume_provisioner_enabled: false +# local_volume_provisioner_namespace: "{{ system_namespace }}" +# local_volume_provisioner_base_dir: /mnt/disks +# local_volume_provisioner_mount_dir: /local-disks +# local_volume_provisioner_storage_class: local-storage # CephFS provisioner deployment cephfs_provisioner_enabled: false diff --git a/docs/local-storage-provisioner.md b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md similarity index 64% rename from docs/local-storage-provisioner.md rename to roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md index 9895cc473..458a483cb 100644 --- a/docs/local-storage-provisioner.md +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md @@ -1,58 +1,62 @@ -# Local Storage Provisioner +Local Storage Provisioner +========================= The local storage provisioner is NOT a dynamic storage provisioner as you would expect from a cloud provider. Instead, it simply creates PersistentVolumes for -all manually created volumes located in the directory `local_volume_base_dir`. +all manually created volumes located in the directory `local_volume_provisioner_base_dir`. The default path is /mnt/disks and the rest of this doc will use that path as an example. -## Examples to create local storage volumes +Examples to create local storage volumes +---------------------------------------- ### tmpfs method: - ``` - for vol in vol1 vol2 vol3; do - mkdir /mnt/disks/$vol - mount -t tmpfs -o size=5G $vol /mnt/disks/$vol - done - ``` +``` bash +for vol in vol1 vol2 vol3; do +mkdir /mnt/disks/$vol +mount -t tmpfs -o size=5G $vol /mnt/disks/$vol +done +``` The tmpfs method is not recommended for production because the mount is not persistent and data will be deleted on reboot. ### Mount physical disks - ``` - mkdir /mnt/disks/ssd1 - mount /dev/vdb1 /mnt/disks/ssd1 - ``` +``` bash +mkdir /mnt/disks/ssd1 +mount /dev/vdb1 /mnt/disks/ssd1 +``` Physical disks are recommended for production environments because it offers complete isolation in terms of I/O and capacity. ### File-backed sparsefile method - ``` - truncate /mnt/disks/disk5 --size 2G - mkfs.ext4 /mnt/disks/disk5 - mkdir /mnt/disks/vol5 - mount /mnt/disks/disk5 /mnt/disks/vol5 - ``` +``` bash +truncate /mnt/disks/disk5 --size 2G +mkfs.ext4 /mnt/disks/disk5 +mkdir /mnt/disks/vol5 +mount /mnt/disks/disk5 /mnt/disks/vol5 +``` If you have a development environment and only one disk, this is the best way to limit the quota of persistent volumes. ### Simple directories - ``` - for vol in vol6 vol7 vol8; do - mkdir /mnt/disks/$vol - done - ``` + +``` bash +for vol in vol6 vol7 vol8; do +mkdir /mnt/disks/$vol +done +``` This is also acceptable in a development environment, but there is no capacity management. -## Usage notes +Usage notes +----------- The volume provisioner cannot calculate volume sizes correctly, so you should delete the daemonset pod on the relevant host after creating volumes. The pod @@ -62,6 +66,7 @@ Make sure to make any mounts persist via /etc/fstab or with systemd mounts (for CoreOS/Container Linux). Pods with persistent volume claims will not be able to start if the mounts become unavailable. -## Further reading +Further reading +--------------- -Refer to the upstream docs here: https://github.com/kubernetes-incubator/external-storage/tree/master/local-volume +Refer to the upstream docs here: diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml new file mode 100644 index 000000000..dd2e8a147 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml @@ -0,0 +1,8 @@ +--- +local_volume_provisioner_image_repo: quay.io/external_storage/local-volume-provisioner +local_volume_provisioner_image_tag: v2.0.0 + +local_volume_provisioner_namespace: "{{ system_namespace }}" +local_volume_provisioner_base_dir: /mnt/disks +local_volume_provisioner_mount_dir: /mnt/disks +local_volume_provisioner_storage_class: local-storage diff --git a/roles/kubernetes-apps/local_volume_provisioner/tasks/main.yml b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml similarity index 52% rename from roles/kubernetes-apps/local_volume_provisioner/tasks/main.yml rename to roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml index 9766ea27c..b83e45a20 100644 --- a/roles/kubernetes-apps/local_volume_provisioner/tasks/main.yml +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml @@ -1,8 +1,9 @@ --- + - name: Local Volume Provisioner | Ensure base dir is created on all hosts file: - path: "{{ local_volume_base_dir }}" - ensure: directory + path: "{{ local_volume_provisioner_base_dir }}" + state: directory owner: root group: root mode: 0700 @@ -13,31 +14,32 @@ - name: Local Volume Provisioner | Create addon dir file: path: "{{ kube_config_dir }}/addons/local_volume_provisioner" + state: directory owner: root group: root mode: 0755 - recurse: true - name: Local Volume Provisioner | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.file }}" with_items: - - { name: local-volume-serviceaccount, file: serviceaccount.yml, type, serviceaccount } - - { name: local-volume-clusterrolebinding, file: clusterrolebinding.yml, type, clusterrolebinding } - - { name: local-volume-configmap, file: configmap.yml, type, configmap } - - { name: local-volume-daemonset, file: daemonset.yml, type, daemonset } - register: local_volume_manifests + - { name: local-volume-provisioner-ns, file: local-volume-provisioner-ns.yml, type: ns } + - { name: local-volume-provisioner-sa, file: local-volume-provisioner-sa.yml, type: sa } + - { name: local-volume-provisioner-clusterrolebinding, file: local-volume-provisioner-clusterrolebinding.yml, type, clusterrolebinding } + - { name: local-volume-provisioner-cm, file: local-volume-provisioner-cm.yml, type, cm } + - { name: local-volume-provisioner-ds, file: local-volume-provisioner-ds.yml, type, ds } + - { name: local-volume-provisioner-sc, file: local-volume-provisioner-sc.yml, type, sc } + register: local_volume_provisioner_manifests when: inventory_hostname == groups['kube-master'][0] - - name: Local Volume Provisioner | Apply manifests kube: name: "{{ item.item.name }}" - namespace: "{{ system_namespace }}" + namespace: "{{ local_volume_provisioner_namespace }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.item.file }}" state: "latest" - with_items: "{{ local_volume_manifests.results }}" + with_items: "{{ local_volume_provisioner_manifests.results }}" when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/local_volume_provisioner/templates/clusterrolebinding.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2 similarity index 51% rename from roles/kubernetes-apps/local_volume_provisioner/templates/clusterrolebinding.yml.j2 rename to roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2 index 5097d2607..ab98f1f55 100644 --- a/roles/kubernetes-apps/local_volume_provisioner/templates/clusterrolebinding.yml.j2 +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2 @@ -1,27 +1,28 @@ --- -kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: local-storage-provisioner-pv-binding - namespace: {{ system_namespace }} + name: local-volume-provisioner-system-persistent-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} subjects: - kind: ServiceAccount - name: local-storage-admin - namespace: {{ system_namespace }} + name: local-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} roleRef: kind: ClusterRole name: system:persistent-volume-provisioner apiGroup: rbac.authorization.k8s.io + --- -kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: local-storage-provisioner-node-binding - namespace: {{ system_namespace }} + name: local-volume-provisioner-system-node + namespace: {{ local_volume_provisioner_namespace }} subjects: - kind: ServiceAccount - name: local-storage-admin - namespace: {{ system_namespace }} + name: local-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} roleRef: kind: ClusterRole name: system:node diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 new file mode 100644 index 000000000..8ad76ab2d --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} +data: + storageClassMap: | + {{ local_volume_provisioner_storage_class }}: + hostDir: {{ local_volume_provisioner_base_dir }} + mountDir: {{ local_volume_provisioner_mount_dir }} diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 new file mode 100644 index 000000000..80a74f5f1 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 @@ -0,0 +1,45 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: local-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} + labels: + k8s-app: local-volume-provisioner + version: {{ local_volume_provisioner_image_tag }} +spec: + selector: + matchLabels: + k8s-app: local-volume-provisioner + version: {{ local_volume_provisioner_image_tag }} + template: + metadata: + labels: + k8s-app: local-volume-provisioner + version: {{ local_volume_provisioner_image_tag }} + spec: + serviceAccountName: local-volume-provisioner + containers: + - name: provisioner + image: {{ local_volume_provisioner_image_repo }}:{{ local_volume_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + privileged: true + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: local-volume-provisioner + mountPath: /etc/provisioner/config + readOnly: true + - name: local-volume-provisioner-hostpath-mnt-disks + mountPath: {{ local_volume_provisioner_mount_dir }} + volumes: + - name: local-volume-provisioner + configMap: + name: local-volume-provisioner + - name: local-volume-provisioner-hostpath-mnt-disks + hostPath: + path: {{ local_volume_provisioner_base_dir }} diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2 new file mode 100644 index 000000000..68faacfbc --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ local_volume_provisioner_namespace }} diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2 new file mode 100644 index 000000000..c78a16b60 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 new file mode 100644 index 000000000..bf1f00262 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ local_volume_provisioner_storage_class }} +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer diff --git a/roles/kubernetes-apps/external_provisioner/meta/main.yml b/roles/kubernetes-apps/external_provisioner/meta/main.yml new file mode 100644 index 000000000..3daa461d8 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/meta/main.yml @@ -0,0 +1,8 @@ +--- +dependencies: + - role: kubernetes-apps/external_provisioner/local_volume_provisioner + when: local_volume_provisioner_enabled + tags: + - apps + - local-volume-provisioner + - external-provisioner diff --git a/roles/kubernetes-apps/local_volume_provisioner/defaults/main.yml b/roles/kubernetes-apps/local_volume_provisioner/defaults/main.yml deleted file mode 100644 index d1e1d1d69..000000000 --- a/roles/kubernetes-apps/local_volume_provisioner/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -local_volume_provisioner_bootstrap_image_repo: quay.io/external_storage/local-volume-provisioner-bootstrap -local_volume_provisioner_bootstrap_image_tag: v1.0.1 - -local_volume_provisioner_image_repo: quay.io/external_storage/local-volume-provisioner -local_volume_provisioner_image_tag: v1.0.1 diff --git a/roles/kubernetes-apps/local_volume_provisioner/templates/configmap.yml.j2 b/roles/kubernetes-apps/local_volume_provisioner/templates/configmap.yml.j2 deleted file mode 100644 index fd8a7a637..000000000 --- a/roles/kubernetes-apps/local_volume_provisioner/templates/configmap.yml.j2 +++ /dev/null @@ -1,14 +0,0 @@ ---- -# The config map is used to configure local volume discovery for Local SSDs on GCE and GKE. -# It is a map from storage class to its mount configuration. -kind: ConfigMap -apiVersion: v1 -metadata: - name: local-volume-config - namespace: {{ system_namespace }} -data: - "{{ local_volume_storage_class }}": | - { - "hostDir": "{{ local_volume_base_dir }}", - "mountDir": "{{ local_volume_mount_dir }}" - } diff --git a/roles/kubernetes-apps/local_volume_provisioner/templates/daemonset.yml.j2 b/roles/kubernetes-apps/local_volume_provisioner/templates/daemonset.yml.j2 deleted file mode 100644 index 6ffe5e36b..000000000 --- a/roles/kubernetes-apps/local_volume_provisioner/templates/daemonset.yml.j2 +++ /dev/null @@ -1,45 +0,0 @@ ---- -kind: DaemonSet -apiVersion: extensions/v1beta1 -metadata: - name: local-volume-provisioner - namespace: "{{ system_namespace }}" -spec: - template: - metadata: - labels: - app: local-volume-provisioner - spec: - containers: - - name: provisioner - image: {{ local_volume_provisioner_image_repo }}:{{ local_volume_provisioner_image_tag }} - imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - privileged: true - volumeMounts: - - name: discovery-vol - mountPath: "{{ local_volume_mount_dir }}" - - name: local-volume-config - mountPath: /etc/provisioner/config/ - env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: MY_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: VOLUME_CONFIG_NAME - value: "local-volume-config" - volumes: - - name: discovery-vol - hostPath: - path: "{{ local_volume_base_dir }}" - - configMap: - defaultMode: 420 - name: local-volume-config - name: local-volume-config - serviceAccount: local-storage-admin diff --git a/roles/kubernetes-apps/local_volume_provisioner/templates/serviceaccount.yml.j2 b/roles/kubernetes-apps/local_volume_provisioner/templates/serviceaccount.yml.j2 deleted file mode 100644 index 182248a6a..000000000 --- a/roles/kubernetes-apps/local_volume_provisioner/templates/serviceaccount.yml.j2 +++ /dev/null @@ -1,5 +0,0 @@ ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: local-storage-admin diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 4f657bd27..fa3b1f1a6 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -27,13 +27,6 @@ dependencies: - apps - registry - - role: kubernetes-apps/local_volume_provisioner - when: local_volume_provisioner_enabled - tags: - - apps - - local_volume_provisioner - - storage - - role: kubernetes-apps/cephfs_provisioner when: cephfs_provisioner_enabled tags: diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 index 6549a7044..28a109ec1 100644 --- a/roles/kubernetes/node/templates/kubelet-container.j2 +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -27,7 +27,7 @@ -v {{kube_config_dir}}:{{kube_config_dir}}:ro \ -v /etc/os-release:/etc/os-release:ro \ {% if local_volume_provisioner_enabled == true %} - -v {{ local_volume_base_dir }}:{{ local_volume_base_dir }}:shared \ + -v {{ local_volume_provisioner_base_dir }}:{{ local_volume_provisioner_base_dir }}:shared \ {% endif %} {{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \ ./hyperkube kubelet \ diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index e1406e7e1..80825fab3 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -37,7 +37,7 @@ ExecStart=/usr/bin/rkt run \ --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false,recursive=true \ --volume var-log,kind=host,source=/var/log \ {% if local_volume_provisioner_enabled == true %} - --volume local-volume-base-dir,kind=host,source={{ local_volume_base_dir }},readOnly=false,recursive=true \ + --volume local-volume-provisioner-base-dir,kind=host,source={{ local_volume_provisioner_base_dir }},readOnly=false,recursive=true \ {% endif %} {% if kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"] %} --volume etc-cni,kind=host,source=/etc/cni,readOnly=true \ @@ -68,7 +68,7 @@ ExecStart=/usr/bin/rkt run \ --mount volume=var-log,target=/var/log \ --mount volume=hosts,target=/etc/hosts \ {% if local_volume_provisioner_enabled == true %} - --mount volume=local-volume-base-dir,target={{ local_volume_base_dir }} \ + --mount volume=local-volume-provisioner-base-dir,target={{ local_volume_provisioner_base_dir }} \ {% endif %} {% if kubelet_flexvolumes_plugins_dir is defined %} --mount volume=flexvolumes,target={{ kubelet_flexvolumes_plugins_dir }} \ diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 289065c71..76f9bfde4 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -60,7 +60,15 @@ - "{{ kube_config_dir }}/ssl" - "{{ kube_manifest_dir }}" - "{{ kube_script_dir }}" - - "{{ local_volume_base_dir }}" + +- name: Create local volume provisioner base dir + file: + path: "{{ local_volume_provisioner_base_dir }}" + state: directory + owner: kube + when: + - inventory_hostname in groups['k8s-cluster'] + - local_volume_provisioner_enabled == true - name: check cloud_provider value fail: diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 6883370ee..d85265eca 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -166,15 +166,10 @@ helm_enabled: false istio_enabled: false registry_enabled: false enable_network_policy: false -local_volume_provisioner_enabled: false +local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}" persistent_volumes_enabled: false cephfs_provisioner_enabled: false -# Base path for local volume provisioner addon -local_volume_base_dir: /mnt/disks -local_volume_mount_dir: /local-disks -local_volume_storage_class: local-storage - ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) # openstack_blockstorage_version: "v1/v2/auto (default)" ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables. @@ -199,7 +194,11 @@ rbac_enabled: "{{ 'RBAC' in authorization_modes or kubeadm_enabled }}" ## List of key=value pairs that describe feature gates for ## the k8s cluster. -kube_feature_gates: ['Initializers={{ istio_enabled|string }}', 'PersistentLocalVolumes={{ local_volume_provisioner_enabled|string }}'] +kube_feature_gates: + - "Initializers={{ istio_enabled | string }}" + - "PersistentLocalVolumes={{ local_volume_provisioner_enabled | string }}" + - "VolumeScheduling={{ local_volume_provisioner_enabled | string }}" + - "MountPropagation={{ local_volume_provisioner_enabled | string }}" # Vault data dirs. vault_base_dir: /etc/vault