Support multiple local volume provisioner StorageClasses (#3450)

- Local Volume StorageClass configuration is now manged by `local_volume_provisioner_storage_classes`, a list of maps that specifies local storage classes with `name` `host_dir` and `mount_dir` keys per entry
- Tasks and templates updated to loop through local volume storage classes
- Previous defaults for path/class names were not changed
- Fixed an issue where a `kubernetes/preinstall` was creating directories inconsistently with the `kubernetes-apps/external_provisioner/local_volume_provisioner` task

inventory/sample/group_vars/k8s-cluster/addons.yml

@@ -14,9 +14,19 @@ registry_enabled: false
 # Local volume provisioner deployment
 local_volume_provisioner_enabled: false
 # local_volume_provisioner_namespace: kube-system
-# local_volume_provisioner_base_dir: /mnt/disks
-# local_volume_provisioner_mount_dir: /mnt/disks
-# local_volume_provisioner_storage_class: local-storage
+# local_volume_provisioner_storage_classes:
+#   - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}"
+#     host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}"
+#     mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
+#   - name: "local-ssd"
+#     host_dir: "/mnt/local-storage/ssd"
+#     mount_dir: "/mnt/local-storage/ssd"
+#   - name: "local-hdd"
+#     host_dir: "/mnt/local-storage/hdd"
+#     mount_dir: "/mnt/local-storage/hdd"
+#   - name: "local-shared"
+#     host_dir: "/mnt/local-storage/shared"
+#     mount_dir: "/mnt/local-storage/shared"
 
 # CephFS provisioner deployment
 cephfs_provisioner_enabled: false

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md

@@ -3,7 +3,7 @@ Local Storage Provisioner
 
 The local storage provisioner is NOT a dynamic storage provisioner as you would
 expect from a cloud provider. Instead, it simply creates PersistentVolumes for
-all manually created volumes located in the directory `local_volume_provisioner_base_dir`.
+all manually created volumes located in the directories specified in the `local_volume_provisioner_storage_classes.host_dir` entries.
 The default path is /mnt/disks and the rest of this doc will use that path as
 an example.
 

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml

@@ -1,5 +1,6 @@
 ---
 local_volume_provisioner_namespace: "kube-system"
-local_volume_provisioner_base_dir: /mnt/disks
-local_volume_provisioner_mount_dir: /mnt/disks
-local_volume_provisioner_storage_class: local-storage
+local_volume_provisioner_storage_classes:
+  - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}"
+    host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}"
+    mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml

@@ -2,13 +2,15 @@
 
 - name: Local Volume Provisioner | Ensure base dir is created on all hosts
   file:
-    path: "{{ local_volume_provisioner_base_dir }}"
+    path: "{{ item[1].host_dir }}"
     state: directory
     owner: root
     group: root
     mode: 0700
-  delegate_to: "{{ item }}"
-  with_items: "{{ groups['k8s-cluster'] }}"
+  delegate_to: "{{ item[0] }}"
+  with_nested:
+    - "{{ groups['k8s-cluster'] }}"
+    - "{{ local_volume_provisioner_storage_classes }}"
   failed_when: false
 
 - name: Local Volume Provisioner | Create addon dir

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2

@@ -6,6 +6,8 @@ metadata:
   namespace: {{ local_volume_provisioner_namespace }}
 data:
   storageClassMap: |
-    {{ local_volume_provisioner_storage_class }}:
-      hostDir: {{ local_volume_provisioner_base_dir }}
-      mountDir: {{ local_volume_provisioner_mount_dir }}
+{% for class in local_volume_provisioner_storage_classes %}
+    {{ class.name }}:
+      hostDir: {{ class.host_dir }}
+      mountDir: {{ class.mount_dir }}
+{% endfor %}

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2

@@ -44,13 +44,17 @@ spec:
             - name: local-volume-provisioner
               mountPath: /etc/provisioner/config
               readOnly: true
-            - name: local-volume-provisioner-hostpath-mnt-disks
-              mountPath: {{ local_volume_provisioner_mount_dir }}
+{% for class in local_volume_provisioner_storage_classes %}
+            - name: {{ class.name }}
+              mountPath: {{ class.mount_dir }}
               mountPropagation: "HostToContainer"
+{% endfor %}
       volumes:
         - name: local-volume-provisioner
           configMap:
             name: local-volume-provisioner
-        - name: local-volume-provisioner-hostpath-mnt-disks
+{% for class in local_volume_provisioner_storage_classes %}
+        - name: {{ class.name }}
           hostPath:
-            path: {{ local_volume_provisioner_base_dir }}
+            path: {{ class.host_dir }}
+{% endfor %}

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2

@@ -25,8 +25,10 @@ spec:
     - 'downwardAPI'
     - 'hostPath'
   allowedHostPaths:
-    - pathPrefix: "{{ local_volume_provisioner_base_dir }}"
+{% for class in local_volume_provisioner_storage_classes %}
+    - pathPrefix: "{{ class.host_dir }}"
       readOnly: false
+{% endfor %}
   hostNetwork: false
   hostIPC: false
   hostPID: false

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2

@@ -1,7 +1,9 @@
+{% for class in local_volume_provisioner_storage_classes %}
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: {{ local_volume_provisioner_storage_class }}
+  name: {{ class.name }}
 provisioner: kubernetes.io/no-provisioner
 volumeBindingMode: WaitForFirstConsumer
+{% endfor %}

roles/kubernetes/node/templates/kubelet-container.j2

@@ -31,8 +31,10 @@
   -v {{ kubelet_flexvolumes_plugins_dir }}:{{ kubelet_flexvolumes_plugins_dir }}:rw \
   {% endif -%}
   {% if local_volume_provisioner_enabled -%}
-  -v {{ local_volume_provisioner_base_dir }}:{{ local_volume_provisioner_base_dir }}:rw \
-  -v {{ local_volume_provisioner_mount_dir }}:{{ local_volume_provisioner_mount_dir }}:rw \
+  {% for class in local_volume_provisioner_storage_classes -%}
+  -v {{ class.host_dir }}:{{ class.host_dir }}:rw \
+  -v {{ class.mount_dir }}:{{ class.mount_dir }}:rw \
+  {% endfor -%}
   {% endif %}
   -v {{kube_config_dir}}:{{kube_config_dir}}:ro \
   -v /etc/os-release:/etc/os-release:ro \

roles/kubernetes/node/templates/kubelet.rkt.service.j2

@@ -47,11 +47,13 @@ ExecStart=/usr/bin/rkt run \
         --volume flexvolumes,kind=host,source={{ kubelet_flexvolumes_plugins_dir }},readOnly=false \
 {% endif -%}
 {% if local_volume_provisioner_enabled %}
-        --volume local-volume-provisioner-base-dir,kind=host,source={{ local_volume_provisioner_base_dir }},readOnly=false \
+{% for class in local_volume_provisioner_storage_classes %}
+        --volume local-volume-provisioner-base-dir,kind=host,source={{ class.host_dir }},readOnly=false \
 {# Not pretty, but needed to avoid double mount #}
-{% if local_volume_provisioner_base_dir not in local_volume_provisioner_mount_dir and local_volume_provisioner_mount_dir not in local_volume_provisioner_base_dir %}
-        --volume local-volume-provisioner-mount-dir,kind=host,source={{ local_volume_provisioner_mount_dir }},readOnly=false \
+{% if class.host_dir not in class.mount_dir and class.mount_dir not in class.host_dir %}
+        --volume local-volume-provisioner-mount-dir,kind=host,source={{ class.mount_dir }},readOnly=false \
 {% endif %}
+{% endfor %}
 {% endif %}
 {% if kubelet_load_modules == true %}
         --mount volume=lib-modules,target=/lib/modules \
@@ -81,11 +83,13 @@ ExecStart=/usr/bin/rkt run \
         --mount volume=flexvolumes,target={{ kubelet_flexvolumes_plugins_dir }} \
 {% endif -%}
 {% if local_volume_provisioner_enabled %}
-        --mount volume=local-volume-provisioner-base-dir,target={{ local_volume_provisioner_base_dir }} \
+{% for class in local_volume_provisioner_storage_classes %}
+        --mount volume=local-volume-provisioner-base-dir,target={{ class.host_dir }} \
 {# Not pretty, but needed to avoid double mount #}
-{% if local_volume_provisioner_base_dir not in local_volume_provisioner_mount_dir and local_volume_provisioner_mount_dir not in local_volume_provisioner_base_dir %}
-        --mount volume=local-volume-provisioner-mount-dir,target={{ local_volume_provisioner_mount_dir }} \
+{% if class.host_dir not in class.mount_dir and class.mount_dir not in class.host_dir %}
+        --mount volume=local-volume-provisioner-mount-dir,target={{ class.mount_dir }} \
 {% endif %}
+{% endfor %}
 {% endif %}
         --stage1-from-dir=stage1-fly.aci \
 {% if kube_hyperkube_image_repo == "docker" %}

roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -46,12 +46,12 @@
 
 - name: Create local volume provisioner directories
   file:
-    path: "{{ item }}"
+    path: "{{ item.host_dir }}"
     state: directory
-    owner: kube
-  with_items:
-    - "{{ local_volume_provisioner_base_dir }}"
-    - "{{ local_volume_provisioner_mount_dir }}"
+    owner: root
+    group: root
+    mode: 0700
+  with_items: "{{ local_volume_provisioner_storage_classes }}"
   when:
     - inventory_hostname in groups['k8s-cluster']
     - local_volume_provisioner_enabled

roles/kubespray-defaults/defaults/main.yaml

@@ -301,9 +301,11 @@ vault_config_dir: "{{ vault_base_dir }}/config"
 vault_roles_dir: "{{ vault_base_dir }}/roles"
 vault_secrets_dir: "{{ vault_base_dir }}/secrets"
 
-# Local volume provisioner dirs
-local_volume_provisioner_base_dir: /mnt/disks
-local_volume_provisioner_mount_dir: /mnt/disks
+# Local volume provisioner storage classes
+local_volume_provisioner_storage_classes:
+  - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}"
+    host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}"
+    mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
 
 # weave's network password for encryption
 # if null then no network encryption