From e97c65ce5fb83db1b00b42cdacbf16294b18889c Mon Sep 17 00:00:00 2001 From: gjmzj Date: Sun, 17 Jan 2021 23:59:40 +0800 Subject: [PATCH] feat: add client kubeconfig management in 'ezctl' --- example/config.yml | 7 +- ezctl | 144 +++++++++++++--- roles/deploy/files/read-group-rbac.yaml | 158 ------------------ .../tasks/add-custom-kubectl-kubeconfig.yml | 46 +++++ .../tasks/create-kubectl-kubeconfig.yml | 28 +--- roles/deploy/tasks/main.yml | 5 + roles/deploy/templates/ca-config.json.j2 | 10 ++ roles/deploy/templates/crb.yaml.j2 | 16 ++ .../{read-csr.json.j2 => user-csr.json.j2} | 4 +- roles/deploy/vars/main.yml | 4 + roles/kube-node/tasks/main.yml | 15 -- roles/kube-node/templates/kubelet.service.j2 | 4 +- 12 files changed, 219 insertions(+), 222 deletions(-) delete mode 100644 roles/deploy/files/read-group-rbac.yaml create mode 100644 roles/deploy/tasks/add-custom-kubectl-kubeconfig.yml create mode 100644 roles/deploy/templates/crb.yaml.j2 rename roles/deploy/templates/{read-csr.json.j2 => user-csr.json.j2} (79%) diff --git a/example/config.yml b/example/config.yml index 0fc1638..3219863 100644 --- a/example/config.yml +++ b/example/config.yml @@ -26,12 +26,9 @@ local_network: "0.0.0.0/0" CA_EXPIRY: "876000h" CERT_EXPIRY: "438000h" -# kubeconfig 配置参数，注意权限根据‘USER_NAME’设置： -# 'admin' 表示创建集群管理员（所有）权限的 kubeconfig -# 'read' 表示创建只读权限的 kubeconfig +# kubeconfig 配置参数 CLUSTER_NAME: "cluster1" -USER_NAME: "admin" -CONTEXT_NAME: "context-{{ CLUSTER_NAME }}-{{ USER_NAME }}" +CONTEXT_NAME: "context-{{ CLUSTER_NAME }}" ############################ diff --git a/ezctl b/ezctl index bd480ca..bd28663 100755 --- a/ezctl +++ b/ezctl @@ -30,29 +30,13 @@ Cluster ops: del-master to delete a master node from the k8s cluster del-node to delete a work node from the k8s cluster +Extra operation: + kcfg-adm to manage client kubeconfig of the k8s cluster + Use "ezctl help " for more information about a given command. EOF } -function usage-setup(){ - echo -e "\033[33mUsage:\033[0m ezctl setup " - cat < 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-node.md'" ;; + (kcfg-adm) + usage-kcfg-adm + ;; (*) echo -e "todo: help info $1" ;; esac } +function usage-kcfg-adm(){ + echo -e "\033[33mUsage:\033[0m ezctl kcfg-adm " + cat <: + -A to add a client kubeconfig with a newly created user + -D to delete a client kubeconfig with the existed user + -L to list all of the users + -e to set expiry of the user certs in hours (ex. 24h, 8h, 240h) + -t to set a user-type (admin or view) + -u to set a user-name prefix + +examples: ./ezctl kcfg-adm test-k8s -L + ./ezctl kcfg-adm default -A -e 240h -t admin -u jack + ./ezctl kcfg-adm default -D -u jim-202101162141 +EOF +} + +function usage-setup(){ + echo -e "\033[33mUsage:\033[0m ezctl setup " + cat <&2; exit 2; } start-aio ;; + ### extra operations ############################## + (kcfg-adm) + [ "$#" -gt 2 ] || { usage-kcfg-adm >&2; exit 2; } + kcfg-adm "${@:2}" + ;; (help) [ "$#" -gt 1 ] || { usage >&2; exit 2; } help-info "$2" diff --git a/roles/deploy/files/read-group-rbac.yaml b/roles/deploy/files/read-group-rbac.yaml deleted file mode 100644 index eb60bad..0000000 --- a/roles/deploy/files/read-group-rbac.yaml +++ /dev/null @@ -1,158 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: read-clusterrole-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: read-clusterrole -subjects: -- kind: Group - name: "group:read" - apiGroup: rbac.authorization.k8s.io - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: read-clusterrole -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - persistentvolumes - - persistentvolumeclaims - - persistentvolumeclaims/status - - pods - - replicationcontrollers - - replicationcontrollers/scale - - serviceaccounts - - services - - services/status - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - bindings - - events - - limitranges - - namespaces/status - - pods/log - - pods/status - - replicationcontrollers/status - - resourcequotas - - resourcequotas/status - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch -- apiGroups: - - apps - resources: - - controllerrevisions - - daemonsets - - daemonsets/status - - deployments - - deployments/scale - - deployments/status - - replicasets - - replicasets/scale - - replicasets/status - - statefulsets - - statefulsets/scale - - statefulsets/status - verbs: - - get - - list - - watch -- apiGroups: - - autoscaling - resources: - - horizontalpodautoscalers - - horizontalpodautoscalers/status - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - cronjobs - - cronjobs/status - - jobs - - jobs/status - verbs: - - get - - list - - watch -- apiGroups: - - extensions - resources: - - daemonsets - - daemonsets/status - - deployments - - deployments/scale - - deployments/status - - ingresses - - ingresses/status - - replicasets - - replicasets/scale - - replicasets/status - - replicationcontrollers/scale - verbs: - - get - - list - - watch -- apiGroups: - - policy - resources: - - poddisruptionbudgets - - poddisruptionbudgets/status - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - - ingresses/status - - networkpolicies - verbs: - - get - - list - - watch -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - - volumeattachments - verbs: - - get - - list - - watch -- apiGroups: - - rbac.authorization.k8s.io - resources: - - clusterrolebindings - - clusterroles - - roles - - rolebindings - verbs: - - get - - list - - watch diff --git a/roles/deploy/tasks/add-custom-kubectl-kubeconfig.yml b/roles/deploy/tasks/add-custom-kubectl-kubeconfig.yml new file mode 100644 index 0000000..0c3ac36 --- /dev/null +++ b/roles/deploy/tasks/add-custom-kubectl-kubeconfig.yml @@ -0,0 +1,46 @@ +- name: 创建自定义用户证书目录 + file: name={{ cluster_dir }}/ssl/users/ state=directory + +- name: 准备CA配置文件 + template: src=ca-config.json.j2 dest={{ cluster_dir }}/ssl/ca-config.json + +- name: 准备kubectl使用的{{ USER_NAME }}证书签名请求 + template: src=user-csr.json.j2 dest={{ cluster_dir }}/ssl/users/{{ USER_NAME }}-csr.json + +- name: 创建{{ USER_NAME }}证书与私钥 + shell: "cd {{ cluster_dir }}/ssl/users && {{ base_dir }}/bin/cfssl gencert \ + -ca={{ cluster_dir }}/ssl/ca.pem \ + -ca-key={{ cluster_dir }}/ssl/ca-key.pem \ + -config={{ cluster_dir }}/ssl/ca-config.json \ + -profile=kcfg {{ USER_NAME }}-csr.json | {{ base_dir }}/bin/cfssljson -bare {{ USER_NAME }}" + +- name: 设置集群参数 + shell: "{{ base_dir }}/bin/kubectl config set-cluster {{ CLUSTER_NAME }} \ + --certificate-authority={{ cluster_dir }}/ssl/ca.pem \ + --embed-certs=true \ + --server={{ KUBE_APISERVER }} \ + --kubeconfig={{ cluster_dir }}/ssl/users/{{ USER_NAME }}.kubeconfig" + +- name: 设置客户端认证参数 + shell: "{{ base_dir }}/bin/kubectl config set-credentials {{ USER_NAME }} \ + --client-certificate={{ cluster_dir }}/ssl/users/{{ USER_NAME }}.pem \ + --embed-certs=true \ + --client-key={{ cluster_dir }}/ssl/users/{{ USER_NAME }}-key.pem \ + --kubeconfig={{ cluster_dir }}/ssl/users/{{ USER_NAME }}.kubeconfig" + +- name: 设置上下文参数 + shell: "{{ base_dir }}/bin/kubectl config set-context {{ CONTEXT_NAME }} \ + --cluster={{ CLUSTER_NAME }} --user={{ USER_NAME }} \ + --kubeconfig={{ cluster_dir }}/ssl/users/{{ USER_NAME }}.kubeconfig" + +- name: 选择默认上下文 + shell: "{{ base_dir }}/bin/kubectl config use-context {{ CONTEXT_NAME }} \ + --kubeconfig={{ cluster_dir }}/ssl/users/{{ USER_NAME }}.kubeconfig" + +- name: 生成clusterrolebind 配置文件 + template: src=crb.yaml.j2 dest={{ cluster_dir }}/ssl/users/crb-{{ USER_NAME }}.yaml + +- name: 创建clusterrolebind 配置 + shell: "{{ base_dir }}/bin/kubectl apply -f {{ cluster_dir }}/ssl/users/crb-{{ USER_NAME }}.yaml" + +- debug: msg="查看{{ USER_NAME }}自定义kubeconfig：{{ cluster_dir }}/ssl/users/{{ USER_NAME }}.kubeconfig" diff --git a/roles/deploy/tasks/create-kubectl-kubeconfig.yml b/roles/deploy/tasks/create-kubectl-kubeconfig.yml index aae3944..0d6c74f 100644 --- a/roles/deploy/tasks/create-kubectl-kubeconfig.yml +++ b/roles/deploy/tasks/create-kubectl-kubeconfig.yml @@ -1,24 +1,12 @@ -- name: 删除原有kubeconfig - file: path=/root/.kube/config state=absent - ignore_errors: true +- name: 准备kubectl使用的admin证书签名请求 + template: src=admin-csr.json.j2 dest={{ cluster_dir }}/ssl/admin-csr.json -- name: 下载 group:read rbac 文件 - copy: src=read-group-rbac.yaml dest=/tmp/read-group-rbac.yaml - when: USER_NAME == "read" - -- name: 创建group:read rbac 绑定 - shell: "{{ base_dir }}/bin/kubectl apply -f /tmp/read-group-rbac.yaml" - when: USER_NAME == "read" - -- name: 准备kubectl使用的{{ USER_NAME }}证书签名请求 - template: src={{ USER_NAME }}-csr.json.j2 dest={{ cluster_dir }}/ssl/{{ USER_NAME }}-csr.json - -- name: 创建{{ USER_NAME }}证书与私钥 +- name: 创建admin证书与私钥 shell: "cd {{ cluster_dir }}/ssl && {{ base_dir }}/bin/cfssl gencert \ -ca=ca.pem \ -ca-key=ca-key.pem \ -config=ca-config.json \ - -profile=kubernetes {{ USER_NAME }}-csr.json | {{ base_dir }}/bin/cfssljson -bare {{ USER_NAME }}" + -profile=kubernetes admin-csr.json | {{ base_dir }}/bin/cfssljson -bare admin" - name: 设置集群参数 shell: "{{ base_dir }}/bin/kubectl config set-cluster {{ CLUSTER_NAME }} \ @@ -28,15 +16,15 @@ --kubeconfig={{ cluster_dir }}/kubectl.kubeconfig" - name: 设置客户端认证参数 - shell: "{{ base_dir }}/bin/kubectl config set-credentials {{ USER_NAME }} \ - --client-certificate={{ cluster_dir }}/ssl/{{ USER_NAME }}.pem \ + shell: "{{ base_dir }}/bin/kubectl config set-credentials admin \ + --client-certificate={{ cluster_dir }}/ssl/admin.pem \ --embed-certs=true \ - --client-key={{ cluster_dir }}/ssl/{{ USER_NAME }}-key.pem \ + --client-key={{ cluster_dir }}/ssl/admin-key.pem \ --kubeconfig={{ cluster_dir }}/kubectl.kubeconfig" - name: 设置上下文参数 shell: "{{ base_dir }}/bin/kubectl config set-context {{ CONTEXT_NAME }} \ - --cluster={{ CLUSTER_NAME }} --user={{ USER_NAME }} \ + --cluster={{ CLUSTER_NAME }} --user=admin \ --kubeconfig={{ cluster_dir }}/kubectl.kubeconfig" - name: 选择默认上下文 diff --git a/roles/deploy/tasks/main.yml b/roles/deploy/tasks/main.yml index 0d1623f..30c55a1 100644 --- a/roles/deploy/tasks/main.yml +++ b/roles/deploy/tasks/main.yml @@ -31,6 +31,11 @@ - import_tasks: create-kubectl-kubeconfig.yml tags: create_kctl_cfg +#----------- 创建个性化客户端配置文件 +- import_tasks: add-custom-kubectl-kubeconfig.yml + tags: add-kcfg + when: "ADD_KCFG|bool" + #------------创建配置文件: kube-proxy.kubeconfig - import_tasks: create-kube-proxy-kubeconfig.yml diff --git a/roles/deploy/templates/ca-config.json.j2 b/roles/deploy/templates/ca-config.json.j2 index f558359..15d5ed2 100644 --- a/roles/deploy/templates/ca-config.json.j2 +++ b/roles/deploy/templates/ca-config.json.j2 @@ -13,6 +13,16 @@ ], "expiry": "{{ CERT_EXPIRY }}" } + }, + "profiles": { + "kcfg": { + "usages": [ + "signing", + "key encipherment", + "client auth" + ], + "expiry": "{{ CUSTOM_EXPIRY }}" + } } } } diff --git a/roles/deploy/templates/crb.yaml.j2 b/roles/deploy/templates/crb.yaml.j2 new file mode 100644 index 0000000..c707741 --- /dev/null +++ b/roles/deploy/templates/crb.yaml.j2 @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: crb-{{ USER_NAME }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{% if USER_TYPE == 'admin' %} + name: cluster-admin +{% else %} + name: view +{% endif %} +subjects: +- kind: User + name: {{ USER_NAME }} + apiGroup: rbac.authorization.k8s.io diff --git a/roles/deploy/templates/read-csr.json.j2 b/roles/deploy/templates/user-csr.json.j2 similarity index 79% rename from roles/deploy/templates/read-csr.json.j2 rename to roles/deploy/templates/user-csr.json.j2 index e70221d..8a9d842 100644 --- a/roles/deploy/templates/read-csr.json.j2 +++ b/roles/deploy/templates/user-csr.json.j2 @@ -1,5 +1,5 @@ { - "CN": "read", + "CN": "{{ USER_NAME }}", "hosts": [], "key": { "algo": "rsa", @@ -10,7 +10,7 @@ "C": "CN", "ST": "HangZhou", "L": "XS", - "O": "group:read", + "O": "k8s", "OU": "System" } ] diff --git a/roles/deploy/vars/main.yml b/roles/deploy/vars/main.yml index 3e9ca34..d1e0eaa 100644 --- a/roles/deploy/vars/main.yml +++ b/roles/deploy/vars/main.yml @@ -1,2 +1,6 @@ # apiserver 默认第一个master节点 KUBE_APISERVER: "https://{{ groups['kube-master'][0] }}:6443" + +# +ADD_KCFG: false +CUSTOM_EXPIRY: "438000h" diff --git a/roles/kube-node/tasks/main.yml b/roles/kube-node/tasks/main.yml index f48d32e..efc8103 100644 --- a/roles/kube-node/tasks/main.yml +++ b/roles/kube-node/tasks/main.yml @@ -33,21 +33,6 @@ - name: 准备 cni配置文件 template: src=cni-default.conf.j2 dest=/etc/cni/net.d/10-default.conf -# 判断 kubernetes 版本 -- name: 注册变量 TMP_VER - shell: "{{ base_dir }}/bin/kube-apiserver --version|cut -d' ' -f2|cut -d'v' -f2" - register: TMP_VER - connection: local - tags: upgrade_k8s, restart_node - -- name: 获取 kubernetes 主版本号 - set_fact: - KUBE_VER: "{{ TMP_VER.stdout.split('.')[0]|int + TMP_VER.stdout.split('.')[1]|int/100 }}" - tags: upgrade_k8s, restart_node - -- name: debug info - debug: var="KUBE_VER" - - name: 创建kubelet的配置文件 template: src=kubelet-config.yaml.j2 dest=/var/lib/kubelet/config.yaml tags: upgrade_k8s, restart_node diff --git a/roles/kube-node/templates/kubelet.service.j2 b/roles/kube-node/templates/kubelet.service.j2 index 5a4d0a2..b384cc9 100644 --- a/roles/kube-node/templates/kubelet.service.j2 +++ b/roles/kube-node/templates/kubelet.service.j2 @@ -29,9 +29,6 @@ ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/system.slice {% endif %} ExecStart={{ bin_dir }}/kubelet \ --config=/var/lib/kubelet/config.yaml \ -{% if KUBE_VER|float < 1.13 %} - --allow-privileged=true \ -{% endif %} --cni-bin-dir={{ bin_dir }} \ --cni-conf-dir=/etc/cni/net.d \ {% if CONTAINER_RUNTIME == "containerd" %} @@ -39,6 +36,7 @@ ExecStart={{ bin_dir }}/kubelet \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ {% endif %} --hostname-override={{ inventory_hostname }} \ + --image-pull-progress-deadline=5m \ --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \ --network-plugin=cni \ --pod-infra-container-image={{ SANDBOX_IMAGE }} \