From c0e989b17c691dffaa5b9cb1a315c3f033a44ba0 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 1 Nov 2017 14:25:35 +0000 Subject: [PATCH] New addon: local_volume_provisioner (#1909) --- docs/local-storage-provisioner.md | 67 +++++++++++++++++++ inventory/group_vars/k8s-cluster.yml | 5 +- .../defaults/main.yml | 6 ++ .../local_volume_provisioner/tasks/main.yml | 42 ++++++++++++ .../provisioner-admin-account.yml.j2 | 34 ++++++++++ .../templates/provisioner-ds.yml.j2 | 42 ++++++++++++ .../templates/volume-config.yml.j2 | 12 ++++ roles/kubernetes-apps/meta/main.yml | 8 +++ .../node/templates/kubelet-container.j2 | 1 + .../node/templates/kubelet.rkt.service.j2 | 2 + roles/kubespray-defaults/defaults/main.yaml | 8 ++- tests/files/centos7-flannel-addons.yml | 1 + 12 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 docs/local-storage-provisioner.md create mode 100644 roles/kubernetes-apps/local_volume_provisioner/defaults/main.yml create mode 100644 roles/kubernetes-apps/local_volume_provisioner/tasks/main.yml create mode 100644 roles/kubernetes-apps/local_volume_provisioner/templates/provisioner-admin-account.yml.j2 create mode 100644 roles/kubernetes-apps/local_volume_provisioner/templates/provisioner-ds.yml.j2 create mode 100644 roles/kubernetes-apps/local_volume_provisioner/templates/volume-config.yml.j2 diff --git a/docs/local-storage-provisioner.md b/docs/local-storage-provisioner.md new file mode 100644 index 000000000..9895cc473 --- /dev/null +++ b/docs/local-storage-provisioner.md @@ -0,0 +1,67 @@ +# Local Storage Provisioner + +The local storage provisioner is NOT a dynamic storage provisioner as you would +expect from a cloud provider. Instead, it simply creates PersistentVolumes for +all manually created volumes located in the directory `local_volume_base_dir`. +The default path is /mnt/disks and the rest of this doc will use that path as +an example. + +## Examples to create local storage volumes + +### tmpfs method: + + ``` + for vol in vol1 vol2 vol3; do + mkdir /mnt/disks/$vol + mount -t tmpfs -o size=5G $vol /mnt/disks/$vol + done + ``` + +The tmpfs method is not recommended for production because the mount is not +persistent and data will be deleted on reboot. + +### Mount physical disks + + ``` + mkdir /mnt/disks/ssd1 + mount /dev/vdb1 /mnt/disks/ssd1 + ``` + +Physical disks are recommended for production environments because it offers +complete isolation in terms of I/O and capacity. + +### File-backed sparsefile method + + ``` + truncate /mnt/disks/disk5 --size 2G + mkfs.ext4 /mnt/disks/disk5 + mkdir /mnt/disks/vol5 + mount /mnt/disks/disk5 /mnt/disks/vol5 + ``` + +If you have a development environment and only one disk, this is the best way +to limit the quota of persistent volumes. + +### Simple directories + ``` + for vol in vol6 vol7 vol8; do + mkdir /mnt/disks/$vol + done + ``` + +This is also acceptable in a development environment, but there is no capacity +management. + +## Usage notes + +The volume provisioner cannot calculate volume sizes correctly, so you should +delete the daemonset pod on the relevant host after creating volumes. The pod +will be recreated and read the size correctly. + +Make sure to make any mounts persist via /etc/fstab or with systemd mounts (for +CoreOS/Container Linux). Pods with persistent volume claims will not be +able to start if the mounts become unavailable. + +## Further reading + +Refer to the upstream docs here: https://github.com/kubernetes-incubator/external-storage/tree/master/local-volume diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index 6473c8d32..09f736af0 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -151,9 +151,12 @@ efk_enabled: false # Helm deployment helm_enabled: false -# Istio depoyment +# Istio deployment istio_enabled: false +# Local volume provisioner deployment +local_volumes_enabled: false + # Make a copy of kubeconfig on the host that runs Ansible in GITDIR/artifacts # kubeconfig_localhost: false # Download kubectl onto the host that runs Ansible in GITDIR/artifacts diff --git a/roles/kubernetes-apps/local_volume_provisioner/defaults/main.yml b/roles/kubernetes-apps/local_volume_provisioner/defaults/main.yml new file mode 100644 index 000000000..b29c15849 --- /dev/null +++ b/roles/kubernetes-apps/local_volume_provisioner/defaults/main.yml @@ -0,0 +1,6 @@ +--- +local_volume_provisioner_bootstrap_image_repo: quay.io/external_storage/local-volume-provisioner-bootstrap +local_volume_provisioner_bootstrap_image_tag: v1.0.0 + +local_volume_provisioner_image_repo: quay.io/external_storage/local-volume-provisioner +local_volume_provisioner_image_tag: v1.0.0 diff --git a/roles/kubernetes-apps/local_volume_provisioner/tasks/main.yml b/roles/kubernetes-apps/local_volume_provisioner/tasks/main.yml new file mode 100644 index 000000000..4e590d964 --- /dev/null +++ b/roles/kubernetes-apps/local_volume_provisioner/tasks/main.yml @@ -0,0 +1,42 @@ +--- +- name: Local Volume Provisioner | Ensure base dir is created on all hosts + file: + path: "{{ local_volume_base_dir }}" + ensure: directory + owner: root + group: root + mode: 0700 + delegate_to: "{{ item }}" + with_items: "{{ groups['k8s-cluster'] }}" + failed_when: false + +- name: Local Volume Provisioner | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/local_volume_provisioner" + owner: root + group: root + mode: 0755 + recurse: true + +- name: Local Volume Provisioner | Create manifests + template: + src: "{{item.file}}.j2" + dest: "{{kube_config_dir}}/addons/local_volume_provisioner/{{item.file}}" + with_items: + - {name: local-storage-provisioner-pv-binding, file: provisioner-admin-account.yml, type: clusterrolebinding} + - {name: local-volume-config, file: volume-config.yml, type: configmap} + - {name: local-volume-provisioner, file: provisioner-ds.yml, type: daemonset} + register: local_volume_manifests + when: inventory_hostname == groups['kube-master'][0] + + +- name: Local Volume Provisioner | Apply manifests + kube: + name: "{{item.item.name}}" + namespace: "{{ system_namespace }}" + kubectl: "{{bin_dir}}/kubectl" + resource: "{{item.item.type}}" + filename: "{{kube_config_dir}}/addons/local_volume_provisioner/{{item.item.file}}" + state: "latest" + with_items: "{{ local_volume_manifests.results }}" + when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/local_volume_provisioner/templates/provisioner-admin-account.yml.j2 b/roles/kubernetes-apps/local_volume_provisioner/templates/provisioner-admin-account.yml.j2 new file mode 100644 index 000000000..5c5c2eb51 --- /dev/null +++ b/roles/kubernetes-apps/local_volume_provisioner/templates/provisioner-admin-account.yml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-storage-admin +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: local-storage-provisioner-pv-binding + namespace: default +subjects: +- kind: ServiceAccount + name: local-storage-admin + namespace: default +roleRef: + kind: ClusterRole + name: system:persistent-volume-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: local-storage-provisioner-node-binding + namespace: default +subjects: +- kind: ServiceAccount + name: local-storage-admin + namespace: default +roleRef: + kind: ClusterRole + name: system:node + apiGroup: rbac.authorization.k8s.io + diff --git a/roles/kubernetes-apps/local_volume_provisioner/templates/provisioner-ds.yml.j2 b/roles/kubernetes-apps/local_volume_provisioner/templates/provisioner-ds.yml.j2 new file mode 100644 index 000000000..302b17a62 --- /dev/null +++ b/roles/kubernetes-apps/local_volume_provisioner/templates/provisioner-ds.yml.j2 @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: local-volume-provisioner + namespace: "{{ system_namespace }}" +spec: + template: + metadata: + labels: + app: local-volume-provisioner + spec: + containers: + - name: provisioner + image: {{ local_volume_provisioner_image_repo }}:{{ local_volume_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + privileged: true + volumeMounts: + - name: discovery-vol + mountPath: "/local-disks" + - name: local-volume-config + mountPath: /etc/provisioner/config/ + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: MY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumes: + - name: discovery-vol + hostPath: + path: "{{ local_volume_base_dir }}" + - configMap: + defaultMode: 420 + name: local-volume-config + name: local-volume-config + serviceAccount: local-storage-admin diff --git a/roles/kubernetes-apps/local_volume_provisioner/templates/volume-config.yml.j2 b/roles/kubernetes-apps/local_volume_provisioner/templates/volume-config.yml.j2 new file mode 100644 index 000000000..97a61fe5c --- /dev/null +++ b/roles/kubernetes-apps/local_volume_provisioner/templates/volume-config.yml.j2 @@ -0,0 +1,12 @@ +# The config map is used to configure local volume discovery for Local SSDs on GCE and GKE. +# It is a map from storage class to its mount configuration. +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-volume-config + namespace: {{ system_namespace }} +data: + storageClassMap: | + local-storage: + hostDir: "{{ local_volume_base_dir }}" + mountDir: "/mnt/local-storage/" diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 13aa11e75..1cd093b33 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -20,6 +20,14 @@ dependencies: tags: - apps - helm + - role: kubernetes-apps/local_volume_provisioner + when: local_volumes_enabled + tags: + - apps + - local_volume_provisioner + - storage + # istio role should be last because it takes a long time to initialize and + # will cause timeouts trying to start other addons. - role: kubernetes-apps/istio when: istio_enabled tags: diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 index 94c7f79a5..1f5212dca 100644 --- a/roles/kubernetes/node/templates/kubelet-container.j2 +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -26,6 +26,7 @@ -v /var/run:/var/run:rw \ -v {{kube_config_dir}}:{{kube_config_dir}}:ro \ -v /etc/os-release:/etc/os-release:ro \ + -v {{ local_volume_base_dir }}:{{ local_volume_base_dir }}:shared \ {{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \ ./hyperkube kubelet \ "$@" diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index db7a4845c..f602319f2 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -32,6 +32,7 @@ ExecStart=/usr/bin/rkt run \ --volume etc-cni,kind=host,source=/etc/cni,readOnly=true \ --volume opt-cni,kind=host,source=/opt/cni,readOnly=true \ --volume var-lib-cni,kind=host,source=/var/lib/cni,readOnly=false \ + --volume local-volume-base-dir,target {{ local_volume_base_dir }},readOnly=false,recursive=true \ --mount volume=etc-cni,target=/etc/cni \ --mount volume=opt-cni,target=/opt/cni \ --mount volume=var-lib-cni,target=/var/lib/cni \ @@ -49,6 +50,7 @@ ExecStart=/usr/bin/rkt run \ --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ --mount volume=var-log,target=/var/log \ --mount volume=hosts,target=/etc/hosts \ + --mount volume=local-volume-base-dir,target={{ local_volume_base_dir }} \ --stage1-from-dir=stage1-fly.aci \ {% if kube_hyperkube_image_repo == "docker" %} --insecure-options=image \ diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 3e69c6ddc..17d769ab8 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -136,10 +136,16 @@ kubectl_localhost: false # K8s image pull policy (imagePullPolicy) k8s_image_pull_policy: IfNotPresent + +# Addons which can be enabled efk_enabled: false helm_enabled: false istio_enabled: false enable_network_policy: false +local_volumes_enabled: false + +# Base path for local volume provisioner addon +local_volume_base_dir: /mnt/disks ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (https://github.com/kubernetes/kubernetes/issues/50461) # openstack_blockstorage_version: "v1/v2/auto (default)" @@ -160,7 +166,7 @@ rbac_enabled: "{{ 'RBAC' in authorization_modes or kubeadm_enabled }}" ## List of key=value pairs that describe feature gates for ## the k8s cluster. -kube_feature_gates: ['Initializers=true'] +kube_feature_gates: ['Initializers=true', 'PersistentLocalVolumes={{ local_volumes_enabled|string }}'] # Vault data dirs. vault_base_dir: /etc/vault diff --git a/tests/files/centos7-flannel-addons.yml b/tests/files/centos7-flannel-addons.yml index 70da8d13e..8824df4a1 100644 --- a/tests/files/centos7-flannel-addons.yml +++ b/tests/files/centos7-flannel-addons.yml @@ -9,6 +9,7 @@ kube_network_plugin: flannel helm_enabled: true istio_enabled: true efk_enabled: true +local_volumes_enabled: true deploy_netchecker: true kubedns_min_replicas: 1 cloud_provider: gce