修改kubelet/docker使用Cgroup driver: systemd

manifests/metrics-server/auth-delegator.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: metrics-server:system:auth-delegator
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:auth-delegator
-subjects:
-- kind: ServiceAccount
-  name: metrics-server
-  namespace: kube-system

manifests/metrics-server/auth-reader.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: metrics-server-auth-reader
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: extension-apiserver-authentication-reader
-subjects:
-- kind: ServiceAccount
-  name: metrics-server
-  namespace: kube-system

manifests/metrics-server/components.yaml

@@ -0,0 +1,153 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:aggregated-metrics-reader
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+rules:
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods", "nodes"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+spec:
+  service:
+    name: metrics-server
+    namespace: kube-system
+  group: metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+spec:
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  template:
+    metadata:
+      name: metrics-server
+      labels:
+        k8s-app: metrics-server
+    spec:
+      serviceAccountName: metrics-server
+      volumes:
+      # mount in tmp so we can safely use from-scratch images and/or read-only containers
+      - name: tmp-dir
+        emptyDir: {}
+      containers:
+      - name: metrics-server
+        #image: k8s.gcr.io/metrics-server-amd64:v0.3.6
+        image: mirrorgooglecontainers/metrics-server-amd64:v0.3.6
+        imagePullPolicy: IfNotPresent
+        args:
+          - --cert-dir=/tmp
+          - --secure-port=4443
+          - --kubelet-insecure-tls
+        ports:
+        - name: main-port
+          containerPort: 4443
+          protocol: TCP
+        securityContext:
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 1000
+        volumeMounts:
+        - name: tmp-dir
+          mountPath: /tmp
+      nodeSelector:
+        kubernetes.io/os: linux
+        kubernetes.io/arch: "amd64"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    kubernetes.io/name: "Metrics-server"
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    k8s-app: metrics-server
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: main-port
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - nodes/stats
+  - namespaces
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system

manifests/metrics-server/metrics-apiservice.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: apiregistration.k8s.io/v1
-kind: APIService
-metadata:
-  name: v1beta1.metrics.k8s.io
-spec:
-  service:
-    name: metrics-server
-    namespace: kube-system
-  group: metrics.k8s.io
-  version: v1beta1
-  insecureSkipTLSVerify: true
-  groupPriorityMinimum: 100
-  versionPriority: 100

manifests/metrics-server/metrics-server-deployment.yaml

@@ -1,41 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: metrics-server
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: metrics-server
-  namespace: kube-system
-  labels:
-    k8s-app: metrics-server
-spec:
-  selector:
-    matchLabels:
-      k8s-app: metrics-server
-  template:
-    metadata:
-      name: metrics-server
-      labels:
-        k8s-app: metrics-server
-    spec:
-      serviceAccountName: metrics-server
-      volumes:
-      # mount in tmp so we can safely use from-scratch images and/or read-only containers
-      - name: tmp-dir
-        emptyDir: {}
-      containers:
-      - name: metrics-server
-        #image: k8s.gcr.io/metrics-server-amd64:v0.3.6
-        image: mirrorgooglecontainers/metrics-server-amd64:v0.3.6
-        imagePullPolicy: IfNotPresent 
-        command:
-        - /metrics-server
-        - --metric-resolution=30s
-        - --kubelet-insecure-tls
-        volumeMounts:
-        - name: tmp-dir
-          mountPath: /tmp

manifests/metrics-server/metrics-server-service.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: metrics-server
-  namespace: kube-system
-  labels:
-    kubernetes.io/name: "Metrics-server"
-    kubernetes.io/cluster-service: "true"
-spec:
-  selector:
-    k8s-app: metrics-server
-  ports:
-  - port: 443
-    protocol: TCP
-    targetPort: 443

manifests/metrics-server/resource-reader.yaml

@@ -1,30 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: system:metrics-server
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - nodes
-  - nodes/stats
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: system:metrics-server
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:metrics-server
-subjects:
-- kind: ServiceAccount
-  name: metrics-server
-  namespace: kube-system

roles/docker/defaults/main.yml

@@ -1,10 +1,3 @@
-# docker日志相关
-
-LOG_DRIVER: "json-file"
-LOG_LEVEL: "warn"
-LOG_MAX_SIZE: "10m"
-LOG_MAX_FILE: 3
-
 # docker容器存储目录
 STORAGE_DIR: "/var/lib/docker"
 

roles/docker/templates/daemon.json.j2

@@ -1,4 +1,6 @@
 {
+  "data-root": "{{ STORAGE_DIR }}",
+  "exec-opts": ["native.cgroupdriver=systemd"],
 {% if ENABLE_MIRROR_REGISTRY %}
   "registry-mirrors": {{ REG_MIRRORS }}, 
 {% endif %}
@@ -7,11 +9,11 @@
 {% endif %}
   "insecure-registries": {{ INSECURE_REG }},
   "max-concurrent-downloads": 10,
-  "log-driver": "{{ LOG_DRIVER }}",
+  "log-driver": "json-file",
-  "log-level": "{{ LOG_LEVEL }}",
+  "log-level": "warn",
   "log-opts": {
-    "max-size": "{{ LOG_MAX_SIZE }}",
+    "max-size": "15m",
-    "max-file": "{{ LOG_MAX_FILE }}"
+    "max-file": "3"
     },
-  "data-root": "{{ STORAGE_DIR }}"
+  "storage-driver": "overlay2"
 }

roles/kube-node/defaults/main.yml

@@ -3,7 +3,6 @@ CLUSTER_DNS_SVC_IP: "{{ SERVICE_CIDR | ipaddr('net') | ipaddr(2) | ipaddr('addre
 
 # 基础容器镜像
 SANDBOX_IMAGE: "easzlab/pause-amd64:3.2"
-#SANDBOX_IMAGE: "registry.access.redhat.com/rhel7/pod-infrastructure:latest"
 
 # Kubelet 根目录
 KUBELET_ROOT_DIR: "/var/lib/kubelet"
@@ -12,18 +11,16 @@ KUBELET_ROOT_DIR: "/var/lib/kubelet"
 MAX_PODS: 110
 
 # 配置为kube组件（kubelet,kube-proxy,dockerd等）预留的资源量
+# 数值设置详见templates/kubelet-config.yaml.j2
 KUBE_RESERVED_ENABLED: "yes"
-KUBE_RESERVED: "{'cpu':'200m','memory':'500Mi','ephemeral-storage':'1Gi'}"
-# k8s 官方不建议草率开启 system-reserved, 除非你基于长期监控，了解系统的资源占用状况；并且随着系统运行时间，需要适当增加资源预留
-SYS_RESERVED_ENABLED: "no"
-# 以下系统预留设置基于 4c/8g 虚机，最小化安装系统服务，如果使用高性能物理机请适当增加数值
-SYS_RESERVED: "{'cpu':'200m','memory':'500Mi','ephemeral-storage':'1Gi'}"
 
-# node 请求 apiserver 负载均衡算法，常见如下：
+# k8s 官方不建议草率开启 system-reserved, 除非你基于长期监控，了解系统的资源占用状况；
-# "roundrobin": 基于服务器权重的轮询
+# 并且随着系统运行时间，需要适当增加资源预留，数值设置详见templates/kubelet-config.yaml.j2
-# "leastconn": 基于服务器最小连接数
+# 系统预留设置基于 4c/8g 虚机，最小化安装系统服务，如果使用高性能物理机可以适当增加预留
-# "source": 基于请求源IP地址
+# 另外，集群安装时候apiserver等资源占用会短时较大，建议至少预留1g内存
-# "uri": 基于请求的URI
+SYS_RESERVED_ENABLED: "no"
+
+# haproxy balance mode 
 BALANCE_ALG: "roundrobin"
 
 # 设置 APISERVER 地址

roles/kube-node/templates/kubelet-config.yaml.j2

@@ -14,7 +14,7 @@ authorization:
   webhook:
     cacheAuthorizedTTL: 5m0s
     cacheUnauthorizedTTL: 30s
-cgroupDriver: cgroupfs
+cgroupDriver: systemd 
 cgroupsPerQOS: true
 clusterDNS:
 - {{ CLUSTER_DNS_SVC_IP }}
@@ -34,7 +34,7 @@ eventBurst: 10
 eventRecordQPS: 5
 evictionHard:
   imagefs.available: 15%
-  memory.available: 200Mi
+  memory.available: 300Mi
   nodefs.available: 10%
   nodefs.inodesFree: 5%
 evictionPressureTransitionPeriod: 5m0s
@@ -48,8 +48,9 @@ imageGCHighThresholdPercent: 85
 imageGCLowThresholdPercent: 80
 imageMinimumGCAge: 2m0s
 {% if KUBE_RESERVED_ENABLED == "yes" %}
-kubeReservedCgroup: /system.slice/kubelet.service
+kubeReservedCgroup: /podruntime.slice
-kubeReserved: {{ KUBE_RESERVED }}
+kubeReserved:
+  memory: 400Mi
 {% endif %}
 kubeAPIBurst: 100
 kubeAPIQPS: 50
@@ -75,7 +76,8 @@ streamingConnectionIdleTimeout: 4h0m0s
 syncFrequency: 1m0s
 {% if SYS_RESERVED_ENABLED == "yes" %}
 systemReservedCgroup: /system.slice
-systemReserved: {{ SYS_RESERVED }}
+systemReserved:
+  memory: 1000Mi
 {% endif %}
 tlsCertFile: {{ ca_dir }}/kubelet.pem
 tlsPrivateKeyFile: {{ ca_dir }}/kubelet-key.pem

roles/kube-node/templates/kubelet.service.j2

@@ -4,12 +4,23 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 
 [Service]
 WorkingDirectory=/var/lib/kubelet
+#ExecStartPre=/bin/mount -o remount,rw '/sys/fs/cgroup'
 {% if KUBE_RESERVED_ENABLED == "yes" or SYS_RESERVED_ENABLED == "yes" %}
-ExecStartPre=/bin/mount -o remount,rw '/sys/fs/cgroup'
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/podruntime.slice
-ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/system.slice/kubelet.service
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/podruntime.slice
-ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/system.slice/kubelet.service
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/podruntime.slice
-ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/system.slice/kubelet.service
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/podruntime.slice
-ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/system.slice/kubelet.service
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/podruntime.slice
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/podruntime.slice
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/podruntime.slice
+
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/system.slice
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/system.slice
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/system.slice
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/system.slice
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/system.slice
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/system.slice
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/system.slice
 {% endif %}
 ExecStart={{ bin_dir }}/kubelet \
   --config=/var/lib/kubelet/config.yaml \

tools/easzup

@@ -16,7 +16,7 @@ set -o errexit
 # default version, can be overridden by cmd line options
 export DOCKER_VER=19.03.8
 export KUBEASZ_VER=2.3.0
-export K8S_BIN_VER=v1.18.2
+export K8S_BIN_VER=v1.18.3
 export EXT_BIN_VER=0.5.0
 export SYS_PKG_VER=0.3.3