From b25e0f82b1cbe7048707c69b4acd324ad62a908c Mon Sep 17 00:00:00 2001 From: Wong Hoi Sing Edison Date: Thu, 1 Feb 2018 12:25:21 +0800 Subject: [PATCH] Add cephfs_provisioner Support for Kubespray --- extra_playbooks/build-cephfs-provisioner.yml | 54 +++++++++++++++++++ inventory/sample/group_vars/k8s-cluster.yml | 11 ++++ .../cephfs_provisioner/defaults/main.yml | 9 ++++ .../cephfs_provisioner/tasks/main.yml | 36 +++++++++++++ .../cephfs-provisioner-clusterrole.yml.j2 | 22 ++++++++ ...phfs-provisioner-clusterrolebinding.yml.j2 | 14 +++++ .../cephfs-provisioner-deploy.yml.j2 | 26 +++++++++ .../templates/cephfs-provisioner-role.yml.j2 | 10 ++++ .../cephfs-provisioner-rolebinding.yml.j2 | 13 +++++ .../templates/cephfs-provisioner-sa.yml.j2 | 6 +++ .../templates/cephfs-provisioner-sc.yml.j2 | 12 +++++ .../cephfs-provisioner-secret.yml.j2 | 9 ++++ roles/kubernetes-apps/meta/main.yml | 7 +++ 13 files changed, 229 insertions(+) create mode 100644 extra_playbooks/build-cephfs-provisioner.yml create mode 100644 roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml create mode 100644 roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml create mode 100644 roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 create mode 100644 roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2 create mode 100644 roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2 create mode 100644 roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2 create mode 100644 roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 create mode 100644 roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2 create mode 100644 roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2 create mode 100644 roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2 diff --git a/extra_playbooks/build-cephfs-provisioner.yml b/extra_playbooks/build-cephfs-provisioner.yml new file mode 100644 index 000000000..6a72a076e --- /dev/null +++ b/extra_playbooks/build-cephfs-provisioner.yml @@ -0,0 +1,54 @@ +--- + +- hosts: localhost + tasks: + - name: CephFS Provisioner | Install pip packages + pip: + name: "{{ item.name }}" + version: "{{ item.version }}" + state: "{{ item.state }}" + with_items: + - { state: "present", name: "docker", version: "2.7.0" } + - { state: "present", name: "docker-compose", version: "1.18.0" } + + - name: CephFS Provisioner | Check Go version + shell: | + go version + ignore_errors: yes + register: go_version_result + + - name: CephFS Provisioner | Install Go 1.9 + shell: | + add-apt-repository -y ppa:gophers/archive + apt-get update + apt-get install -y golang-1.9 + ln -fs /usr/lib/go-1.9/bin/* /usr/local/bin/ + when: 'go_version_result.rc != 0 or "go version go1.9" not in go_version_result.stdout' + + - name: CephFS Provisioner | Check if image exists + shell: | + docker image list | grep 'cephfs-provisioner' + ignore_errors: yes + register: check_image_result + + - block: + - name: CephFS Provisioner | Clone repo + git: + repo: https://github.com/kubernetes-incubator/external-storage.git + dest: "~/go/src/github.com/kubernetes-incubator" + version: 92295a30 + clone: no + update: yes + + - name: CephFS Provisioner | Build image + shell: | + cd ~/go/src/github.com/kubernetes-incubator/external-storage + REGISTRY=quay.io/kubespray/ VERSION=92295a30 make ceph/cephfs + + - name: CephFS Provisioner | Push image + docker_image: + name: quay.io/kubespray/cephfs-provisioner:92295a30 + push: yes + retries: 10 + + when: check_image_result.rc != 0 diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index f3b4ec730..f2782e1da 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -173,6 +173,17 @@ registry_enabled: false local_volumes_enabled: false local_volume_provisioner_enabled: "{{ local_volumes_enabled }}" +# CephFS provisioner deployment +cephfs_provisioner_enabled: false +# cephfs_provisioner_namespace: "{{ system_namespace }}" +# cephfs_provisioner_cluster: ceph +# cephfs_provisioner_monitors: +# - 172.24.0.1:6789 +# - 172.24.0.2:6789 +# - 172.24.0.3:6789 +# cephfs_provisioner_admin_id: admin +# cephfs_provisioner_secret: secret + # Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now ) persistent_volumes_enabled: false diff --git a/roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml b/roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml new file mode 100644 index 000000000..f83edd700 --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml @@ -0,0 +1,9 @@ +--- +cephfs_provisioner_image_repo: quay.io/kubespray/cephfs-provisioner +cephfs_provisioner_image_tag: 92295a30 + +cephfs_provisioner_namespace: "{{ system_namespace }}" +cephfs_provisioner_cluster: ceph +cephfs_provisioner_monitors: [] +cephfs_provisioner_admin_id: admin +cephfs_provisioner_secret: secret diff --git a/roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml b/roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml new file mode 100644 index 000000000..6e854f05e --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml @@ -0,0 +1,36 @@ +--- + +- name: CephFS Provisioner | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/cephfs_provisioner" + owner: root + group: root + mode: 0755 + recurse: true + +- name: CephFS Provisioner | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}" + with_items: + - { name: cephfs-provisioner-sa, file: cephfs-provisioner-sa.yml, type: sa } + - { name: cephfs-provisioner-role, file: cephfs-provisioner-role.yml, type: role } + - { name: cephfs-provisioner-rolebinding, file: cephfs-provisioner-rolebinding.yml, type: rolebinding } + - { name: cephfs-provisioner-clusterrole, file: cephfs-provisioner-clusterrole.yml, type: clusterrole } + - { name: cephfs-provisioner-clusterrolebinding, file: cephfs-provisioner-clusterrolebinding.yml, type: clusterrolebinding } + - { name: cephfs-provisioner-deploy, file: cephfs-provisioner-deploy.yml, type: deploy } + - { name: cephfs-provisioner-secret, file: cephfs-provisioner-secret.yml, type: secret } + - { name: cephfs-provisioner-sc, file: cephfs-provisioner-sc.yml, type: sc } + register: cephfs_manifests + when: inventory_hostname == groups['kube-master'][0] + +- name: CephFS Provisioner | Apply manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ system_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}" + state: "latest" + with_items: "{{ cephfs_manifests.results }}" + when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 new file mode 100644 index 000000000..272db0f70 --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cephfs-provisioner + namespace: {{ system_namespace }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "delete"] diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2 b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2 new file mode 100644 index 000000000..83325f1f8 --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +subjects: + - kind: ServiceAccount + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +roleRef: + kind: ClusterRole + name: cephfs-provisioner + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2 b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2 new file mode 100644 index 000000000..bfe211754 --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2 @@ -0,0 +1,26 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +spec: + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + app: cephfs-provisioner + spec: + containers: + - name: cephfs-provisioner + image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }} + env: + - name: PROVISIONER_NAME + value: ceph.com/cephfs + command: + - "/usr/local/bin/cephfs-provisioner" + args: + - "-id=cephfs-provisioner-1" + serviceAccount: cephfs-provisioner diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2 b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2 new file mode 100644 index 000000000..fb18127f2 --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "delete"] diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 new file mode 100644 index 000000000..f84ed32ba --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +subjects: + - kind: ServiceAccount + name: cephfs-provisioner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cephfs-provisioner diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2 b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2 new file mode 100644 index 000000000..31f87bdc4 --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2 b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2 new file mode 100644 index 000000000..f9e1bc7d3 --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: cephfs +provisioner: ceph.com/cephfs +parameters: + cluster: {{ cephfs_provisioner_cluster }} + monitors: {{ cephfs_provisioner_monitors | join(',') }} + adminId: {{ cephfs_provisioner_admin_id }} + adminSecretName: cephfs-provisioner-{{ cephfs_provisioner_admin_id }}-secret + adminSecretNamespace: {{ cephfs_provisioner_namespace }} diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2 b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2 new file mode 100644 index 000000000..796e30b81 --- /dev/null +++ b/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2 @@ -0,0 +1,9 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: cephfs-provisioner-{{ cephfs_provisioner_admin_id }}-secret + namespace: {{ cephfs_provisioner_namespace }} +type: Opaque +data: + secret: {{ cephfs_provisioner_secret | b64encode }} diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index b1ee4640a..4f657bd27 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -34,6 +34,13 @@ dependencies: - local_volume_provisioner - storage + - role: kubernetes-apps/cephfs_provisioner + when: cephfs_provisioner_enabled + tags: + - apps + - cephfs_provisioner + - storage + # istio role should be last because it takes a long time to initialize and # will cause timeouts trying to start other addons. - role: kubernetes-apps/istio