From e33de18ae39c4510e344cdcf8a6bf604e58de2f1 Mon Sep 17 00:00:00 2001 From: gjmzj Date: Fri, 24 Sep 2021 15:47:32 +0800 Subject: [PATCH] update metrics-server deployment and docs --- docs/guide/metrics-server.md | 27 +- ezdown | 2 +- .../metrics-server/components.yaml.j2 | 271 ++++++++++-------- 3 files changed, 172 insertions(+), 128 deletions(-) diff --git a/docs/guide/metrics-server.md b/docs/guide/metrics-server.md index e964d5e..1535992 100644 --- a/docs/guide/metrics-server.md +++ b/docs/guide/metrics-server.md @@ -8,11 +8,9 @@ - 1.metric-server是扩展的apiserver，依赖于[kube-aggregator](https://github.com/kubernetes/kube-aggregator)，因此需要在apiserver中开启相关参数。 - 2.需要在集群中运行deployment处理请求 -从kubeasz 0.1.0 开始，metrics-server已经默认集成在集群安装脚本中，请查看`roles/cluster-addon/defaults/main.yml`中的设置 +从kubeasz 0.1.0 开始，metrics-server已经默认集成安装，请查看`/etc/kubeasz/clusters/xxxx/config.yml`中的设置 -## 安装 - -默认已集成在90.setup.yml中，如果分步请执行`ansible-play /etc/ansible/07.cluster-addon.yml` +## 前提 - 1.设置apiserver相关[参数](../../roles/kube-master/templates/kube-apiserver.service.j2) ``` bash @@ -31,6 +29,19 @@ 参考1：https://kubernetes.io/docs/tasks/access-kubernetes-api/configure-aggregation-layer/ 参考2：https://kubernetes.io/docs/tasks/access-kubernetes-api/setup-extension-api-server/ +## 安装 + +``` bash +# 默认已经集成安装，假设集群名为xxxx +ezctl setup xxxx all + +# 如果需要分步安装 +ezctl setup xxxx 07 + +# 如果需要手动安装 +kubectl apply -f /etc/kubeasz/clusters/xxxx/yml/metrics-server.yaml +``` + ## 验证 - 查看生成的新api：v1beta1.metrics.k8s.io @@ -50,11 +61,3 @@ $ kubectl top pod --all-namespaces # 输出略 ``` - 验证基于metrics-server实现的基础hpa自动缩放，请参考[hpa.md](hpa.md) - -## 补充 - -目前dashboard插件如果想在界面上显示资源使用率，它还依赖于`heapster`；另外，测试发现k8s 1.8版本的`kubectl top`也依赖`heapster`，因此建议补充安装`heapster`，无需安装`influxdb`和`grafana`。 - -``` bash -$ kubectl apply -f /etc/ansible/manifests/heapster/heapster.yaml -``` diff --git a/ezdown b/ezdown index bb74eaf..6ae2045 100755 --- a/ezdown +++ b/ezdown @@ -28,7 +28,7 @@ dnsNodeCacheVer=1.17.0 corednsVer=1.8.4 dashboardVer=v2.3.1 dashboardMetricsScraperVer=v1.0.6 -metricsVer=v0.3.6 +metricsVer=v0.5.0 pauseVer=3.5 nfsProvisionerVer=v4.0.1 export ciliumVer=v1.4.1 diff --git a/roles/cluster-addon/templates/metrics-server/components.yaml.j2 b/roles/cluster-addon/templates/metrics-server/components.yaml.j2 index 8d79287..f9ba075 100644 --- a/roles/cluster-addon/templates/metrics-server/components.yaml.j2 +++ b/roles/cluster-addon/templates/metrics-server/components.yaml.j2 @@ -1,129 +1,36 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:aggregated-metrics-reader - labels: - rbac.authorization.k8s.io/aggregate-to-view: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-admin: "true" -rules: -- apiGroups: ["metrics.k8s.io"] - resources: ["pods", "nodes"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metrics-server:system:auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: metrics-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - name: v1beta1.metrics.k8s.io -spec: - service: - name: metrics-server - namespace: kube-system - group: metrics.k8s.io - version: v1beta1 - insecureSkipTLSVerify: true - groupPriorityMinimum: 100 - versionPriority: 100 ---- apiVersion: v1 kind: ServiceAccount metadata: - name: metrics-server - namespace: kube-system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metrics-server - namespace: kube-system labels: k8s-app: metrics-server -spec: - selector: - matchLabels: - k8s-app: metrics-server - template: - metadata: - name: metrics-server - labels: - k8s-app: metrics-server - spec: - serviceAccountName: metrics-server - volumes: - # mount in tmp so we can safely use from-scratch images and/or read-only containers - - name: tmp-dir - emptyDir: {} - containers: - - name: metrics-server - #image: k8s.gcr.io/metrics-server-amd64:{{ metricsVer }} - image: mirrorgooglecontainers/metrics-server-amd64:{{ metricsVer }} - imagePullPolicy: IfNotPresent - args: - - --cert-dir=/tmp - - --secure-port=4443 - - --kubelet-insecure-tls - ports: - - name: main-port - containerPort: 4443 - protocol: TCP - securityContext: - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - name: tmp-dir - mountPath: /tmp - nodeSelector: - kubernetes.io/os: linux - kubernetes.io/arch: "amd64" ---- -apiVersion: v1 -kind: Service -metadata: name: metrics-server namespace: kube-system - labels: - kubernetes.io/name: "Metrics-server" - kubernetes.io/cluster-service: "true" -spec: - selector: - k8s-app: metrics-server - ports: - - port: 443 - protocol: TCP - targetPort: main-port --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server name: system:metrics-server rules: - apiGroups: @@ -140,8 +47,41 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server name: system:metrics-server roleRef: apiGroup: rbac.authorization.k8s.io @@ -151,3 +91,104 @@ subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-insecure-tls + - --kubelet-use-node-status-port + - --metric-resolution=15s + #image: k8s.gcr.io/metrics-server/metrics-server:v0.5.0 + image: easzlab/metrics-server:{{ metricsVer }} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + volumes: + - emptyDir: {} + name: tmp-dir +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100