diff --git a/docs/op/config_guide.md b/docs/op/config_guide.md new file mode 100644 index 0000000..bac3a4d --- /dev/null +++ b/docs/op/config_guide.md @@ -0,0 +1,19 @@ +# 个性化集群参数配置 + +`kubeasz`创建集群主要在以下两个地方进行配置: + +- ansible hosts 文件(模板在examples目录):集群主要节点定义和主要参数配置 +- roles/xxx/vars/main.yml 文件:其他参数配置或者部分组件参数配置 + +这些文件都在.gitignore忽略范围,因此修改后项目目录能够保持`git status | clean` + +## ansible hosts + +项目尽量保持`ansible hosts`简单、灵活,在[快速指南](../quickStart.md)或者[集群规划与安装概览](../00-集群规划和基础参数设定.md)已经介绍过,主要包括集群节点定义和集群范围的主要参数配置;目前提供三种集群部署模板。 + +尽量保持配置项稳定。 + +## roles/xxx/vars/main.yml + +主要包括集群某个具体组件的个性化配置,具体组件的配置项可能会不断增加;项目初始时该配置与 roles/xxx/defaults/main.yml 一致,确保在不做任何配置情况下可以使用默认值创建集群;因 ansilbe 变量优先级关系,后续如果对 roles/xxx/vars/main.yml变量修改,那么它将覆盖默认配置。 + diff --git a/manifests/nfs-provisioner/nfs-dynamic-storageclass.yaml b/manifests/nfs-provisioner/nfs-dynamic-storageclass.yaml deleted file mode 100644 index bc505d5..0000000 --- a/manifests/nfs-provisioner/nfs-dynamic-storageclass.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: nfs-dynamic-class -#此处引用nfs-client-provisioner里面的 nfs-prov-1 -provisioner: nfs-prov-1 diff --git a/manifests/storage/alicloud-nas/alicloud-disk.yaml b/manifests/storage/alicloud-nas/alicloud-disk.yaml new file mode 100644 index 0000000..42a9b3d --- /dev/null +++ b/manifests/storage/alicloud-nas/alicloud-disk.yaml @@ -0,0 +1,99 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: alicloud-disk-common +provisioner: alicloud/disk +parameters: + type: cloud +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: alicloud-disk-efficiency +provisioner: alicloud/disk +parameters: + type: cloud_efficiency +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: alicloud-disk-ssd +provisioner: alicloud/disk +parameters: + type: cloud_ssd +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: alicloud-disk-available +provisioner: alicloud/disk +parameters: + type: available +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: alicloud-disk-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alicloud-disk-controller + namespace: kube-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: run-alicloud-disk-controller +subjects: + - kind: ServiceAccount + name: alicloud-disk-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: alicloud-disk-controller-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: alicloud-disk-controller + namespace: kube-system +spec: + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + app: alicloud-disk-controller + spec: + serviceAccount: alicloud-disk-controller + containers: + - name: alicloud-disk-controller + image: registry.cn-hangzhou.aliyuncs.com/acs/alicloud-disk-controller:v1.9.3-ed710ce + volumeMounts: + - name: cloud-config + mountPath: /etc/kubernetes/ + - name: logdir + mountPath: /var/log/alicloud/ + volumes: + - name: cloud-config + hostPath: + path: /etc/kubernetes/ + - name: logdir + hostPath: + path: /var/log/alicloud/ diff --git a/manifests/nfs-provisioner/test/test-claim.yaml b/manifests/storage/test/test-claim.yaml similarity index 80% rename from manifests/nfs-provisioner/test/test-claim.yaml rename to manifests/storage/test/test-claim.yaml index 1956623..92d6344 100644 --- a/manifests/nfs-provisioner/test/test-claim.yaml +++ b/manifests/storage/test/test-claim.yaml @@ -3,7 +3,7 @@ apiVersion: v1 metadata: name: test-claim spec: - storageClassName: nfs-dynamic-class + storageClassName: alicloud-nas accessModes: - ReadWriteMany resources: diff --git a/manifests/nfs-provisioner/test/test-pod.yaml b/manifests/storage/test/test-pod.yaml similarity index 100% rename from manifests/nfs-provisioner/test/test-pod.yaml rename to manifests/storage/test/test-pod.yaml diff --git a/manifests/nfs-provisioner/test/test.yaml b/manifests/storage/test/test.yaml similarity index 93% rename from manifests/nfs-provisioner/test/test.yaml rename to manifests/storage/test/test.yaml index 04eeeb9..a78ef2e 100644 --- a/manifests/nfs-provisioner/test/test.yaml +++ b/manifests/storage/test/test.yaml @@ -3,7 +3,7 @@ apiVersion: v1 metadata: name: test spec: - storageClassName: nfs-dynamic-class + storageClassName: alicloud-nas accessModes: - ReadWriteMany resources: diff --git a/roles/cluster-addon/tasks/main.yml b/roles/cluster-addon/tasks/main.yml index 94d433c..cfee385 100644 --- a/roles/cluster-addon/tasks/main.yml +++ b/roles/cluster-addon/tasks/main.yml @@ -1,12 +1,12 @@ #-------------kube-dns 插件参数初始化 # kubedns.yaml文件中部分参数根据hosts文件设置而定,因此需要用template模块替换参数 - name: 准备 kubedns的部署文件 kubedns.yaml - template: src=kubedns.yaml.j2 dest={{ base_dir }}/manifests/kubedns/kubedns.yaml + template: src=dns/kubedns.yaml.j2 dest={{ base_dir }}/manifests/kubedns/kubedns.yaml when: "hostvars[inventory_hostname]['group_names'].count('deploy') == 1" # coredns.yaml文件中部分参数根据hosts文件设置而定,因此需要用template模块替换参数 - name: 准备 coredns的部署文件 coredns.yaml - template: src=coredns.yaml.j2 dest={{ base_dir }}/manifests/coredns/coredns.yaml + template: src=dns/coredns.yaml.j2 dest={{ base_dir }}/manifests/coredns/coredns.yaml when: "hostvars[inventory_hostname]['group_names'].count('deploy') == 1" - name: 获取所有已经创建的POD信息 diff --git a/roles/cluster-addon/templates/coredns.yaml.j2 b/roles/cluster-addon/templates/dns/coredns.yaml.j2 similarity index 100% rename from roles/cluster-addon/templates/coredns.yaml.j2 rename to roles/cluster-addon/templates/dns/coredns.yaml.j2 diff --git a/roles/cluster-addon/templates/kubedns.yaml.j2 b/roles/cluster-addon/templates/dns/kubedns.yaml.j2 similarity index 100% rename from roles/cluster-addon/templates/kubedns.yaml.j2 rename to roles/cluster-addon/templates/dns/kubedns.yaml.j2 diff --git a/roles/cluster-storage/cluster-storage.yml b/roles/cluster-storage/cluster-storage.yml new file mode 100644 index 0000000..72378b3 --- /dev/null +++ b/roles/cluster-storage/cluster-storage.yml @@ -0,0 +1,8 @@ +- hosts: deploy + roles: + - cluster-storage + vars: + storage_type: nfs + storage_server: 172.16.3.86 + storage_path: /data/nfs + storage_class_name: nfs-dynamic-class diff --git a/roles/cluster-storage/defaults/main.yml b/roles/cluster-storage/defaults/main.yml new file mode 100644 index 0000000..86fbe2d --- /dev/null +++ b/roles/cluster-storage/defaults/main.yml @@ -0,0 +1,5 @@ +# 动态存储类型, 目前支持nfs和alicloud-nas +storage_type: nfs +storage_server: 172.16.3.86 +storage_path: /data/nfs +storage_class_name: nfs-dynamic-class diff --git a/roles/cluster-storage/tasks/main.yml b/roles/cluster-storage/tasks/main.yml new file mode 100644 index 0000000..73d6975 --- /dev/null +++ b/roles/cluster-storage/tasks/main.yml @@ -0,0 +1,27 @@ +- block: + - name: 准备部署nfs-client动态存储 + template: + src: nfs/nfs-client-provisioner.yaml.j2 + dest: "{{ base_dir }}/manifests/storage/nfs/nfs-client-provisioner.yaml" + - name: 开始部署nfs-client动态存储 + shell: "{{ bin_dir }}/kubectl apply -f {{ base_dir }}/manifests/storage/nfs/nfs-client-provisioner.yaml" + when: 'storage_type == "nfs"' + +- block: + - name: 准备部署alicloud-nas动态存储 + template: + src: alicloud-nas/alicloud-nas.yaml.j2 + dest: "{{ base_dir }}/manifests/storage/alicloud-nas/alicloud-nas.yaml" + - name: 开始部署alicloud-disk存储 + shell: "{{ bin_dir }}/kubectl apply -f {{ base_dir }}/manifests/storage/alicloud-nas/alicloud-disk.yaml" + - name: 开始部署alicloud-nas动态存储 + shell: "{{ bin_dir }}/kubectl apply -f {{ base_dir }}/manifests/storage/alicloud-nas/alicloud-nas.yaml" + when: 'storage_type == "alicloud-nas"' + +- block: + - name: 准备部署动态存储类 + template: + src: dynamic-storageclass.yaml.j2 + dest: "{{ base_dir }}/manifests/storage/dynamic-storageclass.yaml" + - name: 开始部署动态存储类 + shell: "{{ bin_dir }}/kubectl apply -f {{ base_dir }}/manifests/storage/dynamic-storageclass.yaml" diff --git a/roles/cluster-storage/templates/alicloud-nas/alicloud-nas.yaml.j2 b/roles/cluster-storage/templates/alicloud-nas/alicloud-nas.yaml.j2 new file mode 100644 index 0000000..ea2d5e4 --- /dev/null +++ b/roles/cluster-storage/templates/alicloud-nas/alicloud-nas.yaml.j2 @@ -0,0 +1,58 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: alicloud-nas +provisioner: alicloud/nas +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alicloud-nas-controller + namespace: kube-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: run-alicloud-nas-controller +subjects: + - kind: ServiceAccount + name: alicloud-nas-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: alicloud-disk-controller-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Deployment +apiVersion: apps/v1beta1 +metadata: + name: alicloud-nas-controller + namespace: kube-system +spec: + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + app: alicloud-nas-controller + spec: + serviceAccount: alicloud-nas-controller + containers: + - name: alicloud-nas-controller + image: registry.cn-hangzhou.aliyuncs.com/acs/alicloud-nas-controller:v1.8.4 + volumeMounts: + - mountPath: /persistentvolumes + name: nfs-client-root + env: + - name: PROVISIONER_NAME + value: alicloud/nas + - name: NFS_SERVER + value: {{ storage_server }} + - name: NFS_PATH + value: {{ storage_path }} + volumes: + - name: nfs-client-root + nfs: + server: {{ storage_server }} + path: {{ storage_path }} diff --git a/roles/cluster-storage/templates/dynamic-storageclass.yaml.j2 b/roles/cluster-storage/templates/dynamic-storageclass.yaml.j2 new file mode 100644 index 0000000..a8a21cd --- /dev/null +++ b/roles/cluster-storage/templates/dynamic-storageclass.yaml.j2 @@ -0,0 +1,5 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ storage_class_name }} +provisioner: prov diff --git a/manifests/nfs-provisioner/nfs-client-provisioner.yaml b/roles/cluster-storage/templates/nfs/nfs-client-provisioner.yaml.j2 similarity index 91% rename from manifests/nfs-provisioner/nfs-client-provisioner.yaml rename to roles/cluster-storage/templates/nfs/nfs-client-provisioner.yaml.j2 index b74bad0..cd3351d 100644 --- a/manifests/nfs-provisioner/nfs-client-provisioner.yaml +++ b/roles/cluster-storage/templates/nfs/nfs-client-provisioner.yaml.j2 @@ -67,15 +67,13 @@ spec: env: - name: PROVISIONER_NAME # 此处供应者名字供storageclass调用 - value: nfs-prov-1 + value: prov - name: NFS_SERVER - value: 10.1.241.230 + value: {{ storage_server }} - name: NFS_PATH - value: /home/share/k8s-pv + value: {{ storage_path }} volumes: - name: nfs-client-root nfs: - server: 10.1.241.230 - path: /home/share/k8s-pv - ---- + server: {{ storage_server }} + path: {{ storage_path }} diff --git a/roles/prepare/files/95-k8s-sysctl.conf b/roles/prepare/files/95-k8s-sysctl.conf index c0277b8..f58fdd0 100644 --- a/roles/prepare/files/95-k8s-sysctl.conf +++ b/roles/prepare/files/95-k8s-sysctl.conf @@ -2,3 +2,4 @@ net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-arptables = 1 +vm.swappiness = 0 diff --git a/roles/prepare/tasks/main.yml b/roles/prepare/tasks/main.yml index b949e89..f2057df 100644 --- a/roles/prepare/tasks/main.yml +++ b/roles/prepare/tasks/main.yml @@ -92,6 +92,18 @@ regexp: 'kubectl completion' line: 'source <(kubectl completion bash)' +# 禁用系统swap +- name: 禁用系统 swap + shell: "swapoff -a && sysctl -w vm.swappiness=0" + ignore_errors: true + +- name: 删除fstab swap 相关配置 + lineinfile: + path: /etc/fstab + regexp: 'swap' + state: absent + backup: 'yes' + # 设置系统参数for k8s # 消除docker info 警告WARNING: bridge-nf-call-ip[6]tables is disabled - name: 设置系统参数 diff --git a/tools/init_vars.yml b/tools/init_vars.yml new file mode 100644 index 0000000..b732a71 --- /dev/null +++ b/tools/init_vars.yml @@ -0,0 +1,32 @@ +# [可选]初始化集群设置脚本,使用请参考docs/op/config_guide.md +# 如果创建集群时需要修改项目roles中默认配置,请执行本脚本后,编辑roles/xxx/vars/main.yml修改 + +- hosts: deploy + tasks: + - name: 创建变量配置目录 roles/xxx/vars + file: name={{ base_dir }}/roles/{{ item }}/vars state=directory + with_items: + - calico + - cluster-addon + - cluster-restore + - flannel + - helm + - kube-node + - kube-router + - lb + + - name: 复制默认配置以备修改 + copy: + src: "{{ base_dir }}/roles/{{ item }}/defaults/main.yml" + dest: "{{ base_dir }}/roles/{{ item }}/vars/main.yml" + force: "no" + with_items: + - calico + - cluster-addon + - cluster-restore + - flannel + - helm + - kube-node + - kube-router + - lb +