From f144b8e4d6afccb22c3e02a37f294bb6bdb4cf8a Mon Sep 17 00:00:00 2001 From: gjmzj Date: Wed, 5 Jan 2022 12:43:03 +0800 Subject: [PATCH] various fixes --- README.md | 9 +- docs/mixes/conformance.md | 96 +++++++++---------- docs/setup/08-cluster-storage.md | 2 +- example/hosts.allinone | 2 +- example/hosts.multi-node | 2 +- ezdown | 2 +- roles/calico/templates/calico-v3.19.yaml.j2 | 8 +- .../nfs-provisioner/nfs-provisioner.yaml.j2 | 5 +- .../templates/kube-apiserver.service.j2 | 3 +- .../templates/kube-proxy-config.yaml.j2 | 6 +- .../templates/kubelet-config.yaml.j2 | 4 +- 11 files changed, 67 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 4e16f3e..68c424f 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,13 @@ 项目致力于提供快速部署高可用`k8s`集群的工具, 同时也努力成为`k8s`实践、使用的参考书；基于二进制方式部署和利用`ansible-playbook`实现自动化；既提供一键安装脚本, 也可以根据`安装指南`分步执行安装各个组件。 - **集群特性** `TLS`双向认证、`RBAC`授权、[多Master高可用](docs/setup/00-planning_and_overall_intro.md#ha-architecture)、支持`Network Policy`、备份恢复、[离线安装](docs/setup/offline_install.md) -- **集群版本** kubernetes v1.19, v1.20, v1.21, v1.22 +- **集群版本** kubernetes v1.20, v1.21, v1.22, v1.23 - **操作系统** CentOS/RedHat 7, Debian 9/10, Ubuntu 16.04/18.04/20.04 -- **运行时** docker 19.03.x, 20.10.x [containerd](docs/setup/containerd.md) v1.4.4 +- **运行时** docker 19.03.x, 20.10.x [containerd](docs/setup/containerd.md) v1.5.8 - **网络** [calico](docs/setup/network-plugin/calico.md), [cilium](docs/setup/network-plugin/cilium.md), [flannel](docs/setup/network-plugin/flannel.md), [kube-ovn](docs/setup/network-plugin/kube-ovn.md), [kube-router](docs/setup/network-plugin/kube-router.md) -**[news]** kubeasz 通过cncf一致性测试 [详情](https://github.com/cncf/k8s-conformance/tree/master/v1.20/kubeasz) +**[news]** kubeasz 通过cncf一致性测试 [详情](docs/mixes/conformance.md) **[news]** 群里大佬上新一套免费[kubernetes架构师课程](https://www.toutiao.com/c/user/token/MS4wLjABAAAA0YFomuMNm87NNysXeUsQdI0Tt3gOgz8WG_0B3MzxsmI/?tab=article)，强烈推荐！ @@ -23,6 +23,7 @@ 1.20 1.21 1.22 + 1.23 @@ -32,6 +33,7 @@ 3.0.1 3.1.0 3.1.1 + 3.2.0 @@ -124,7 +126,6 @@ - 推荐阅读 - [kubernetes-the-hard-way](https://github.com/kelseyhightower/kubernetes-the-hard-way) - [feisky-Kubernetes 指南](https://github.com/feiskyer/kubernetes-handbook/blob/master/SUMMARY.md) - - [rootsongjc-Kubernetes 指南](https://github.com/rootsongjc/kubernetes-handbook) - [opsnull 安装教程](https://github.com/opsnull/follow-me-install-kubernetes-cluster) ## 贡献&致谢 diff --git a/docs/mixes/conformance.md b/docs/mixes/conformance.md index c281b24..075cb67 100644 --- a/docs/mixes/conformance.md +++ b/docs/mixes/conformance.md @@ -1,10 +1,22 @@ -## 关于K8S集群一致性认证 +# 关于K8S集群一致性认证 CNCF 一致性认证项目(https://github.com/cncf/k8s-conformance) 可以很方便帮助k8s搭建者和用户确认集群各项功能符合预期，既符合k8s设计标准。 +# kubeasz 通过一致性测试 + +Cheers! + +自kubeasz 3.0.0 版本，k8s v1.20.2开始，正式通过cncf一致性认证，成为cncf 官方认证安装工具；后续k8s主要版本发布或者kubeasz有大版本更新，会优先确保通过集群一致性认证。 + +v1.23 [进行中]() +v1.22 [已认证](https://github.com/cncf/k8s-conformance/tree/master/v1.22/kubeasz) +v1.21 [已认证](https://github.com/cncf/k8s-conformance/tree/master/v1.21/kubeasz) +v1.20 [已认证](https://github.com/cncf/k8s-conformance/tree/master/v1.20/kubeasz) + + ## Conformance Test -按照测试文档，注意以下几点，通过所有的测试项也不是难事： +按照测试文档，注意以下几点： 1.解决qiang的问题，可以临时去国外公有云创建集群，然后运行测试项目。 @@ -12,93 +24,74 @@ CNCF 一致性认证项目(https://github.com/cncf/k8s-conformance) 可以很方 3.网络组件选择calico，其他组件可能有bug导致特定测试项失败 -4.kube-proxy暂时用iptables模式，使用ipvs再测试服务sessionAffinity时有bug，后续应该会修复 +4.kube-proxy暂时用iptables模式，使用ipvs在测试服务sessionAffinity时有bug，后续应该会修复 -## kubeasz 技术上完全通过一致性测试 +# 附：测试流程 -Cheers! +## Node Provisioning -使用kubeasz 3.0.0 版本，k8s v1.20.2（其他kubeasz版本应该也类似），开始测试时候在网络上走了一些弯路，后面还是很顺利的通过测试，测试结果： - -``` bash -JUnit report was created: /tmp/results/junit_01.xml -{"msg":"Test Suite completed","total":311,"completed":311,"skipped":5356,"failed":0} - -Ran 311 of 5667 Specs in 6179.487 seconds -SUCCESS! -- 311 Passed | 0 Failed | 0 Pending | 5356 Skipped -PASS - -Ginkgo ran 1 suite in 1h43m0.59512776s -Test Suite Passed -``` - -具体的测试过程和结果请参考这里：https://github.com/cncf/k8s-conformance/pull/1326 - -PS：另外，我也花时间走流程正式申请成为官方认证的部署工具；目前来看作为免费的开源工具申请下来还是比较困难，估计是类似的发行版及部署工具太多了吧，中文项目估计也不被看好，有兴趣的或者有门路的朋友可以联系我，帮忙申请下来。 - -后续k8s主要版本发布或者kubeasz有大版本更新，我都会优先确保通过集群一致性认证。 - - -## 附：测试流程 - -### Node Provisioning - -Provision 2 nodes for your cluster (OS requirements: CentOS 7 or Ubuntu 1604/1804) +Provision 3 nodes for your cluster (OS: Ubuntu 20.04) 1 master node (4c16g) -1 worker node (4c16g) +2 worker node (4c16g) for a High-Availability Kubernetes Cluster, read [more](https://github.com/easzlab/kubeasz/blob/master/docs/setup/00-planning_and_overall_intro.md) -### Install the cluster +## Install the cluster -(1) clone repo: kubeasz +(1) Download 'kubeasz' code, the binaries and offline images ``` -git clone https://github.com/easzlab/kubeasz.git -mv ./kubeasz /etc -``` - -(2) Download the binaries and offline images - -``` -cd /etc/kubeasz +export release=3.1.0 +curl -C- -fLO --retry 3 https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown +chmod +x ./ezdown ./ezdown -D -m standard ``` -(3) install an all-in-one cluster +(2) install an all-in-one cluster ``` +cd /etc/kubeasz sed -i 's/^CLUSTER_NETWORK=.*$/CLUSTER_NETWORK="calico"/g' example/hosts.allinone sed -i 's/^PROXY_MODE=.*$/PROXY_MODE="iptables"/g' example/hosts.allinone ./ezdown -S docker exec -it kubeasz ezctl start-aio ``` -(4) Add a worker node +(3) Add two worker nodes ``` -ssh-copy-id ${worker_ip} -docker exec -it kubeasz ezctl add-node default ${worker_ip} +ssh-copy-id ${worker1_ip} +ssh ${worker1_ip} ln -s /usr/bin/python3 /usr/bin/python +docker exec -it kubeasz ezctl add-node default ${worker1_ip} +ssh-copy-id ${worker2_ip} +ssh ${worker2_ip} ln -s /usr/bin/python3 /usr/bin/python +docker exec -it kubeasz ezctl add-node default ${worker2_ip} ``` -### Run Conformance Test -The standard tool for running these tests is Sonobuoy. Sonobuoy is regularly built and kept up to date to execute against all currently supported versions of kubernetes. +## Run Conformance Test -Download a [binary release](https://github.com/vmware-tanzu/sonobuoy/releases) of the CLI, or build it yourself by running: +The standard tool for running these tests is +[Sonobuoy](https://github.com/heptio/sonobuoy). Sonobuoy is +regularly built and kept up to date to execute against all +currently supported versions of kubernetes. + +Download a [binary release](https://github.com/heptio/sonobuoy/releases) of the CLI, or build it yourself by running: ``` -go get -u -v github.com/vmware-tanzu/sonobuoy +$ go get -u -v github.com/heptio/sonobuoy ``` Deploy a Sonobuoy pod to your cluster with: ``` -sonobuoy run --mode=certified-conformance +$ sonobuoy run --mode=certified-conformance ``` +**NOTE:** You can run the command synchronously by adding the flag `--wait` but be aware that running the Conformance tests can take an hour or more. + View actively running pods: ``` @@ -131,3 +124,4 @@ To clean up Kubernetes objects created by Sonobuoy, run: ``` sonobuoy delete ``` + diff --git a/docs/setup/08-cluster-storage.md b/docs/setup/08-cluster-storage.md index a962098..fd87452 100644 --- a/docs/setup/08-cluster-storage.md +++ b/docs/setup/08-cluster-storage.md @@ -40,7 +40,7 @@ spec: 在一个工作k8s 集群中，`PVC`请求会很多，如果每次都需要管理员手动去创建对应的 `PV`资源，那就很不方便；因此 K8S还提供了多种 `provisioner`来动态创建 `PV`，不仅节省了管理员的时间，还可以根据`StorageClasses`封装不同类型的存储供 PVC 选用。 -项目中以nfs-client-provisioner为例（https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner） +项目中以nfs-client-provisioner为例 https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner - 1.编辑集群配置文件：clusters/${集群名}/config.yml diff --git a/example/hosts.allinone b/example/hosts.allinone index 8927374..2c40068 100644 --- a/example/hosts.allinone +++ b/example/hosts.allinone @@ -30,7 +30,7 @@ SECURE_PORT="6443" # Cluster container-runtime supported: docker, containerd -CONTAINER_RUNTIME="docker" +CONTAINER_RUNTIME="containerd" # Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn CLUSTER_NETWORK="flannel" diff --git a/example/hosts.multi-node b/example/hosts.multi-node index 1234546..792fdf2 100644 --- a/example/hosts.multi-node +++ b/example/hosts.multi-node @@ -34,7 +34,7 @@ SECURE_PORT="6443" # Cluster container-runtime supported: docker, containerd -CONTAINER_RUNTIME="docker" +CONTAINER_RUNTIME="containerd" # Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn CLUSTER_NETWORK="flannel" diff --git a/ezdown b/ezdown index 0e4f97c..4bf61c2 100755 --- a/ezdown +++ b/ezdown @@ -30,7 +30,7 @@ dashboardVer=v2.4.0 dashboardMetricsScraperVer=v1.0.7 metricsVer=v0.5.2 pauseVer=3.6 -nfsProvisionerVer=v4.0.1 +nfsProvisionerVer=v4.0.2 export ciliumVer=v1.4.1 export kubeRouterVer=v0.3.1 export kubeOvnVer=v1.5.3 diff --git a/roles/calico/templates/calico-v3.19.yaml.j2 b/roles/calico/templates/calico-v3.19.yaml.j2 index 8df5391..a84d131 100644 --- a/roles/calico/templates/calico-v3.19.yaml.j2 +++ b/roles/calico/templates/calico-v3.19.yaml.j2 @@ -210,7 +210,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.19.2 + image: docker.io/calico/cni:{{ calico_ver }} command: ["/opt/cni/bin/install"] envFrom: - configMapRef: @@ -254,7 +254,7 @@ spec: # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: docker.io/calico/pod2daemon-flexvol:v3.19.2 + image: docker.io/calico/pod2daemon-flexvol:{{ calico_ver }} volumeMounts: - name: flexvol-driver-host mountPath: /host/driver @@ -265,7 +265,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.19.2 + image: docker.io/calico/node:{{ calico_ver }} envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. @@ -514,7 +514,7 @@ spec: hostNetwork: true containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.19.2 + image: docker.io/calico/kube-controllers:{{ calico_ver }} env: # The location of the etcd cluster. - name: ETCD_ENDPOINTS diff --git a/roles/cluster-addon/templates/nfs-provisioner/nfs-provisioner.yaml.j2 b/roles/cluster-addon/templates/nfs-provisioner/nfs-provisioner.yaml.j2 index c8f5942..4259fba 100644 --- a/roles/cluster-addon/templates/nfs-provisioner/nfs-provisioner.yaml.j2 +++ b/roles/cluster-addon/templates/nfs-provisioner/nfs-provisioner.yaml.j2 @@ -11,6 +11,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: nfs-client-provisioner-runner rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] @@ -65,7 +68,6 @@ roleRef: name: leader-locking-nfs-client-provisioner apiGroup: rbac.authorization.k8s.io - --- apiVersion: apps/v1 kind: Deployment @@ -108,7 +110,6 @@ spec: server: {{ nfs_server }} path: {{ nfs_path }} - --- apiVersion: storage.k8s.io/v1 kind: StorageClass diff --git a/roles/kube-master/templates/kube-apiserver.service.j2 b/roles/kube-master/templates/kube-apiserver.service.j2 index 996ccc9..2bffe69 100644 --- a/roles/kube-master/templates/kube-apiserver.service.j2 +++ b/roles/kube-master/templates/kube-apiserver.service.j2 @@ -5,12 +5,11 @@ After=network.target [Service] ExecStart={{ bin_dir }}/kube-apiserver \ - --advertise-address={{ inventory_hostname }} \ --allow-privileged=true \ --anonymous-auth=false \ --api-audiences=api,istio-ca \ --authorization-mode=Node,RBAC \ - --bind-address={{ inventory_hostname }} \ + --bind-address=0.0.0.0 \ --client-ca-file={{ ca_dir }}/ca.pem \ --endpoint-reconciler-type=lease \ --etcd-cafile={{ ca_dir }}/ca.pem \ diff --git a/roles/kube-node/templates/kube-proxy-config.yaml.j2 b/roles/kube-node/templates/kube-proxy-config.yaml.j2 index 18f0d9a..5ef19ed 100644 --- a/roles/kube-node/templates/kube-proxy-config.yaml.j2 +++ b/roles/kube-node/templates/kube-proxy-config.yaml.j2 @@ -1,6 +1,6 @@ kind: KubeProxyConfiguration apiVersion: kubeproxy.config.k8s.io/v1alpha1 -bindAddress: {{ inventory_hostname }} +bindAddress: 0.0.0.0 clientConnection: kubeconfig: "/etc/kubernetes/kube-proxy.kubeconfig" clusterCIDR: "{{ CLUSTER_CIDR }}" @@ -9,7 +9,7 @@ conntrack: min: 131072 tcpCloseWaitTimeout: 1h0m0s tcpEstablishedTimeout: 24h0m0s -healthzBindAddress: {{ inventory_hostname }}:10256 +healthzBindAddress: 0.0.0.0:10256 hostnameOverride: "{{ inventory_hostname }}" -metricsBindAddress: {{ inventory_hostname }}:10249 +metricsBindAddress: 0.0.0.0:10249 mode: "{{ PROXY_MODE }}" diff --git a/roles/kube-node/templates/kubelet-config.yaml.j2 b/roles/kube-node/templates/kubelet-config.yaml.j2 index 8a21a76..727823a 100644 --- a/roles/kube-node/templates/kubelet-config.yaml.j2 +++ b/roles/kube-node/templates/kubelet-config.yaml.j2 @@ -1,6 +1,6 @@ kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 -address: {{ inventory_hostname }} +address: 0.0.0.0 authentication: anonymous: enabled: false @@ -45,7 +45,7 @@ evictionPressureTransitionPeriod: 5m0s failSwapOn: true fileCheckFrequency: 40s hairpinMode: hairpin-veth -healthzBindAddress: {{ inventory_hostname }} +healthzBindAddress: 0.0.0.0 healthzPort: 10248 httpCheckFrequency: 40s imageGCHighThresholdPercent: 85