feat:use 'systemd' cgroup driver

roles/containerd/templates/config.toml.j2

@@ -1,14 +1,26 @@
-root = "{{ CONTAINERD_STORAGE_DIR }}"
+version = 2
+root = "{{ CONTAINERD_STORAGE_DIR }}" 
 state = "/run/containerd"
-oom_score = -999
+plugin_dir = ""
+disabled_plugins = []
+required_plugins = []
+oom_score = 0
 
 [grpc]
   address = "/run/containerd/containerd.sock"
+  tcp_address = ""
+  tcp_tls_cert = ""
+  tcp_tls_key = ""
   uid = 0
   gid = 0
   max_recv_message_size = 16777216
   max_send_message_size = 16777216
 
+[ttrpc]
+  address = ""
+  uid = 0
+  gid = 0
+
 [debug]
   address = ""
   uid = 0
@@ -22,73 +34,109 @@ oom_score = -999
 [cgroup]
   path = ""
 
+[timeouts]
+  "io.containerd.timeout.shim.cleanup" = "5s"
+  "io.containerd.timeout.shim.load" = "5s"
+  "io.containerd.timeout.shim.shutdown" = "3s"
+  "io.containerd.timeout.task.state" = "2s"
+
 [plugins]
-  [plugins.cgroups]
+  [plugins."io.containerd.gc.v1.scheduler"]
-    no_prometheus = false
-  [plugins.cri]
-    stream_server_address = "127.0.0.1"
-    stream_server_port = "0"
-    enable_selinux = false
-    sandbox_image = "{{ SANDBOX_IMAGE }}"
-    stats_collect_period = 10
-    systemd_cgroup = false
-    enable_tls_streaming = false
-    max_container_log_line_size = 16384
-    [plugins.cri.containerd]
-      snapshotter = "overlayfs"
-      no_pivot = false
-      [plugins.cri.containerd.default_runtime]
-        runtime_type = "io.containerd.runtime.v1.linux"
-        runtime_engine = ""
-        runtime_root = ""
-      [plugins.cri.containerd.untrusted_workload_runtime]
-        runtime_type = ""
-        runtime_engine = ""
-        runtime_root = ""
-    [plugins.cri.cni]
-      bin_dir = "{{ bin_dir }}"
-      conf_dir = "/etc/cni/net.d"
-      conf_template = "/etc/cni/net.d/10-default.conf"
-{% if ENABLE_MIRROR_REGISTRY %}
-    [plugins.cri.registry]
-      [plugins.cri.registry.mirrors]
-        # reference from https://www.ilanni.com/?p=14534
-        [plugins.cri.registry.mirrors."docker.io"]
-          endpoint = [
-            "https://docker.mirrors.ustc.edu.cn",
-            "http://hub-mirror.c.163.com"
-          ]
-        [plugins.cri.registry.mirrors."gcr.io"]
-          endpoint = [
-            "https://gcr.mirrors.ustc.edu.cn"
-          ]
-        [plugins.cri.registry.mirrors."k8s.gcr.io"]
-          endpoint = [
-            "https://gcr.mirrors.ustc.edu.cn/google-containers/"
-          ]
-        [plugins.cri.registry.mirrors."quay.io"]
-          endpoint = [
-            "https://quay.mirrors.ustc.edu.cn"
-          ]
-{% endif %}
-    [plugins.cri.x509_key_pair_streaming]
-      tls_cert_file = ""
-      tls_key_file = ""
-  [plugins.diff-service]
-    default = ["walking"]
-  [plugins.linux]
-    shim = "containerd-shim"
-    runtime = "runc"
-    runtime_root = ""
-    no_shim = false
-    shim_debug = false
-  [plugins.opt]
-    path = "/opt/containerd"
-  [plugins.restart]
-    interval = "10s"
-  [plugins.scheduler]
     pause_threshold = 0.02
     deletion_threshold = 0
     mutation_threshold = 100
     schedule_delay = "0s"
     startup_delay = "100ms"
+  [plugins."io.containerd.grpc.v1.cri"]
+    disable_tcp_service = true
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    stream_idle_timeout = "4h0m0s"
+    enable_selinux = false
+    selinux_category_range = 1024
+    sandbox_image = "{{ SANDBOX_IMAGE }}"
+    stats_collect_period = 10
+    systemd_cgroup = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    disable_cgroup = false
+    disable_apparmor = false
+    restrict_oom_score_adj = false
+    max_concurrent_downloads = 3
+    disable_proc_mount = false
+    unset_seccomp_profile = ""
+    tolerate_missing_hugetlb_controller = true
+    disable_hugetlb_controller = true
+    ignore_image_defined_volumes = false
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      snapshotter = "overlayfs"
+      default_runtime_name = "runc"
+      no_pivot = false
+      disable_snapshot_annotations = true
+      discard_unpacked_layers = false
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        runtime_type = ""
+        runtime_engine = ""
+        runtime_root = ""
+        privileged_without_host_devices = false
+        base_runtime_spec = ""
+      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
+        runtime_type = ""
+        runtime_engine = ""
+        runtime_root = ""
+        privileged_without_host_devices = false
+        base_runtime_spec = ""
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          runtime_engine = ""
+          runtime_root = ""
+          privileged_without_host_devices = false
+          base_runtime_spec = ""
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            SystemdCgroup = true
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      bin_dir = "{{ bin_dir }}"
+      conf_dir = "/etc/cni/net.d"
+      max_conf_num = 1
+      conf_template = "/etc/cni/net.d/10-default.conf"
+{% if ENABLE_MIRROR_REGISTRY %}
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+          endpoint = ["https://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"]
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."gcr.io"]
+          endpoint = ["https://gcr.mirrors.ustc.edu.cn"]
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
+          endpoint = ["https://gcr.mirrors.ustc.edu.cn/google-containers/"]
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"]
+          endpoint = ["https://quay.mirrors.ustc.edu.cn"]
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".image_decryption]
+      key_model = ""
+    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+  [plugins."io.containerd.internal.v1.opt"]
+    path = "/opt/containerd"
+  [plugins."io.containerd.internal.v1.restart"]
+    interval = "10s"
+  [plugins."io.containerd.metadata.v1.bolt"]
+    content_sharing_policy = "shared"
+  [plugins."io.containerd.monitor.v1.cgroups"]
+    no_prometheus = false
+  [plugins."io.containerd.runtime.v1.linux"]
+    shim = "containerd-shim"
+    runtime = "runc"
+    runtime_root = ""
+    no_shim = false
+    shim_debug = false
+  [plugins."io.containerd.runtime.v2.task"]
+    platforms = ["linux/amd64"]
+  [plugins."io.containerd.service.v1.diff-service"]
+    default = ["walking"]
+  [plugins."io.containerd.snapshotter.v1.devmapper"]
+    root_path = ""
+    pool_name = ""
+    base_image_size = ""
+    async_remove = false

roles/docker/templates/daemon.json.j2

@@ -1,6 +1,6 @@
 {
   "data-root": "{{ DOCKER_STORAGE_DIR }}",
-  "exec-opts": ["native.cgroupdriver=cgroupfs"],
+  "exec-opts": ["native.cgroupdriver={{ CGROUP_DRIVER }}"],
 {% if ENABLE_MIRROR_REGISTRY %}
   "registry-mirrors": [
     "https://docker.mirrors.ustc.edu.cn",

roles/docker/vars/main.yml

@@ -0,0 +1,6 @@
+# cgroup driver
+CGROUP_DRIVER: "{%- if DOCKER_VER|float >= 20.10 -%} \
+                     systemd \
+                {%- else -%} \
+                     cgroupfs \
+                {%- endif -%}"

roles/kube-node/tasks/main.yml

@@ -36,6 +36,19 @@
 - name: 设置变量 CLUSTER_DNS_SVC_IP
   set_fact: CLUSTER_DNS_SVC_IP={{ DNS_SVC_IP.stdout }}
 
+- block:
+  - name: 获取docker版本信息
+    shell: "{{ base_dir }}/bin/dockerd --version|cut -d' ' -f3"
+    register: docker_ver
+
+  - name: 转换docker版本信息为浮点数
+    set_fact:
+      DOCKER_VER: "{{ docker_ver.stdout.split('.')[0]|int + docker_ver.stdout.split('.')[1]|int/100 }}"
+  connection: local
+  run_once: true
+  tags: upgrade_k8s, restart_node
+  when: "CONTAINER_RUNTIME == 'docker'"
+
 - name: 创建kubelet的配置文件
   template: src=kubelet-config.yaml.j2 dest=/var/lib/kubelet/config.yaml
   tags: upgrade_k8s, restart_node

roles/kube-node/templates/kubelet-config.yaml.j2

@@ -14,7 +14,7 @@ authorization:
   webhook:
     cacheAuthorizedTTL: 5m0s
     cacheUnauthorizedTTL: 30s
-cgroupDriver: cgroupfs 
+cgroupDriver: {{ CGROUP_DRIVER }} 
 cgroupsPerQOS: true
 clusterDNS:
 {% if ENABLE_LOCAL_DNS_CACHE %}

roles/kube-node/vars/main.yml

@@ -3,3 +3,14 @@ KUBE_APISERVER: "https://127.0.0.1:{{ SECURE_PORT }}"
 
 # node local dns cache 离线镜像
 dnscache_offline: "k8s-dns-node-cache_{{ dnsNodeCacheVer }}.tar"
+
+# cgroup driver
+CGROUP_DRIVER: "{%- if CONTAINER_RUNTIME == 'containerd' -%} \
+                     systemd \
+                 {%- else -%} \
+                     {%- if DOCKER_VER|float >= 20.10 -%} \
+                         systemd \
+                     {%- else -%} \
+                         cgroupfs \
+                     {%- endif -%} \
+                 {%- endif -%}"