diff --git a/03.containerd.yml b/03.containerd.yml new file mode 100644 index 0000000..faf4f76 --- /dev/null +++ b/03.containerd.yml @@ -0,0 +1,6 @@ +# to install containerd service +- hosts: + - kube-master + - kube-node + roles: + - containerd diff --git a/90.setup.yml b/90.setup.yml index c9e28cb..ff4e7a7 100644 --- a/90.setup.yml +++ b/90.setup.yml @@ -28,12 +28,13 @@ roles: - etcd -# to install docker service +# to install container runtime - hosts: - kube-master - kube-node roles: - - docker + - { role: docker, when: "CONTAINER_RUNTIME == 'docker'" } + - { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" } # to set up 'kube-master' nodes - hosts: kube-master @@ -68,7 +69,7 @@ - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" } - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" } -# to install clust-addons +# to install cluster-addons - hosts: - kube-node roles: diff --git a/docs/op/change_ip_allinone.md b/docs/op/change_ip_allinone.md index 6bb74a4..d42655d 100644 --- a/docs/op/change_ip_allinone.md +++ b/docs/op/change_ip_allinone.md @@ -8,7 +8,6 @@ ## 操作步骤 前提 ：一个运行正常的allinone部署在虚机，关机后复制给别人使用，新虚机开机后如果需要修改IP，请执行如下步骤： -- 0.拉取最新项目代码：`git pull origin master` - 1.修改ansible hosts文件：`sed -i 's/$OLD_IP/$NEW_IP/g' /etc/ansible/hosts` - 2.配置ssh免密码登陆：`ssh-copy-id $NEW_IP` 按提示完成 - 3.检查下修改是否成功，并且能够成功执行 `ansible all -m ping` diff --git a/example/hosts.allinone.example b/example/hosts.allinone.example index 0831a55..474643a 100644 --- a/example/hosts.allinone.example +++ b/example/hosts.allinone.example @@ -32,6 +32,9 @@ DEPLOY_MODE=allinone MASTER_IP="{{ groups['kube-master'][0] }}" KUBE_APISERVER="https://{{ MASTER_IP }}:6443" +# 集群容器运行时，支持 docker, containerd +CONTAINER_RUNTIME="docker" + # 集群网络插件，目前支持calico, flannel, kube-router, cilium CLUSTER_NETWORK="flannel" diff --git a/example/hosts.allinone.example.en b/example/hosts.allinone.example.en index 95ea154..94dd954 100644 --- a/example/hosts.allinone.example.en +++ b/example/hosts.allinone.example.en @@ -33,6 +33,9 @@ DEPLOY_MODE=allinone MASTER_IP="{{ groups['kube-master'][0] }}" KUBE_APISERVER="https://{{ MASTER_IP }}:6443" +# Cluster container-runtime supported: docker, containerd +CONTAINER_RUNTIME="docker" + # Network plugins supported: calico, flannel, kube-router, cilium CLUSTER_NETWORK="flannel" diff --git a/example/hosts.cloud.example b/example/hosts.cloud.example index ab86381..8716f44 100644 --- a/example/hosts.cloud.example +++ b/example/hosts.cloud.example @@ -32,6 +32,9 @@ DEPLOY_MODE=multi-master MASTER_IP="192.168.1.10" # 即负载均衡内网地址 KUBE_APISERVER="https://{{ MASTER_IP }}:8443" +# 集群容器运行时，支持 docker, containerd +CONTAINER_RUNTIME="docker" + # 集群网络插件，目前支持calico, flannel, kube-router, cilium CLUSTER_NETWORK="flannel" diff --git a/example/hosts.cloud.example.en b/example/hosts.cloud.example.en index 5e0aaa4..b1e073d 100644 --- a/example/hosts.cloud.example.en +++ b/example/hosts.cloud.example.en @@ -33,6 +33,9 @@ DEPLOY_MODE=multi-master MASTER_IP="192.168.1.10" KUBE_APISERVER="https://{{ MASTER_IP }}:8443" +# Cluster container-runtime supported: docker, containerd +CONTAINER_RUNTIME="docker" + # Network plugins supported: calico, flannel CLUSTER_NETWORK="flannel" diff --git a/example/hosts.m-masters.example b/example/hosts.m-masters.example index 7e50eed..54fc885 100644 --- a/example/hosts.m-masters.example +++ b/example/hosts.m-masters.example @@ -42,6 +42,9 @@ DEPLOY_MODE=multi-master MASTER_IP="192.168.1.10" KUBE_APISERVER="https://{{ MASTER_IP }}:8443" +# 集群容器运行时，支持 docker, containerd +CONTAINER_RUNTIME="docker" + # 集群网络插件，目前支持calico, flannel, kube-router, cilium CLUSTER_NETWORK="flannel" diff --git a/example/hosts.m-masters.example.en b/example/hosts.m-masters.example.en index 67075e5..cec81f8 100644 --- a/example/hosts.m-masters.example.en +++ b/example/hosts.m-masters.example.en @@ -43,6 +43,9 @@ DEPLOY_MODE=multi-master MASTER_IP="192.168.1.10" KUBE_APISERVER="https://{{ MASTER_IP }}:8443" +# Cluster container-runtime supported: docker, containerd +CONTAINER_RUNTIME="docker" + # Network plugins supported: calico, flannel, kube-router, cilium CLUSTER_NETWORK="flannel" diff --git a/example/hosts.s-master.example b/example/hosts.s-master.example index e8a2e38..b0f2cfc 100644 --- a/example/hosts.s-master.example +++ b/example/hosts.s-master.example @@ -33,6 +33,9 @@ DEPLOY_MODE=single-master MASTER_IP="{{ groups['kube-master'][0] }}" KUBE_APISERVER="https://{{ MASTER_IP }}:6443" +# 集群容器运行时，支持 docker, containerd +CONTAINER_RUNTIME="docker" + # 集群网络插件，目前支持calico, flannel, kube-router, cilium CLUSTER_NETWORK="flannel" diff --git a/example/hosts.s-master.example.en b/example/hosts.s-master.example.en index aa4a339..9208f3e 100644 --- a/example/hosts.s-master.example.en +++ b/example/hosts.s-master.example.en @@ -34,6 +34,9 @@ DEPLOY_MODE=single-master MASTER_IP="{{ groups['kube-master'][0] }}" KUBE_APISERVER="https://{{ MASTER_IP }}:6443" +# Cluster container-runtime supported: docker, containerd +CONTAINER_RUNTIME="docker" + # Network plugins supported: calico, flannel, kube-router, cilium CLUSTER_NETWORK="flannel" diff --git a/roles/containerd/defaults/main.yml b/roles/containerd/defaults/main.yml new file mode 100644 index 0000000..b81df10 --- /dev/null +++ b/roles/containerd/defaults/main.yml @@ -0,0 +1,10 @@ +# 容器持久化存储目录 +STORAGE_DIR: "/var/lib/containerd" + +# 基础容器镜像 +SANDBOX_IMAGE: "mirrorgooglecontainers/pause-amd64:3.1" + +# 国内镜像加速 +REG_MIRRORS: + - "https://dockerhub.azk8s.cn" + - "https://docker.mirrors.ustc.edu.cn" diff --git a/roles/containerd/tasks/main.yml b/roles/containerd/tasks/main.yml new file mode 100644 index 0000000..646d448 --- /dev/null +++ b/roles/containerd/tasks/main.yml @@ -0,0 +1,56 @@ +- name: 准备containerd相关目录 + file: name={{ item }} state=directory + with_items: + - "{{ bin_dir }}" + - "/etc/containerd" + +- name: 安装 libseccomp2 + package: name=libseccomp2 state=present + when: ansible_distribution == "Ubuntu" or ansible_distribution == "Debian" + +- name: 安装 libseccomp + package: name=libseccomp state=present + when: ansible_distribution == "CentOS" or ansible_distribution == "RedHat" or ansible_distribution == "Amazon" + +- name: 加载内核模块 overlay + modprobe: name=overlay state=present + +- name: 下载 containerd 二进制文件 + copy: src={{ base_dir }}/bin/containerd-bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755 + with_items: + - containerd + - containerd-shim + - containerd-shim-runc-v1 + - containerd-stress + - crictl + - critest + - ctr + - runc + tags: upgrade + +- name: 创建 containerd 配置文件 + template: src=config.toml.j2 dest=/etc/containerd/config.toml + tags: upgrade + +- name: 创建systemd unit文件 + template: src=containerd.service.j2 dest=/etc/systemd/system/containerd.service + tags: upgrade + +- name: 创建 crictl 配置 + template: src=crictl.yaml.j2 dest=/etc/crictl.yaml + +- name: 开机启用 containerd 服务 + shell: systemctl enable containerd + ignore_errors: true + +- name: 开启 containerd 服务 + shell: systemctl daemon-reload && systemctl restart containerd + tags: upgrade + +- name: 轮询等待containerd服务运行 + shell: "systemctl status containerd.service|grep Active" + register: containerd_status + until: '"running" in containerd_status.stdout' + retries: 8 + delay: 2 + tags: upgrade diff --git a/roles/containerd/templates/config.toml.j2 b/roles/containerd/templates/config.toml.j2 new file mode 100644 index 0000000..a76e707 --- /dev/null +++ b/roles/containerd/templates/config.toml.j2 @@ -0,0 +1,81 @@ +root = "{{ STORAGE_DIR }}" +state = "/run/containerd" +oom_score = 0 + +[grpc] + address = "/run/containerd/containerd.sock" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[plugins] + [plugins.cgroups] + no_prometheus = false + [plugins.cri] + stream_server_address = "127.0.0.1" + stream_server_port = "0" + enable_selinux = false + sandbox_image = "{{ SANDBOX_IMAGE }}" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + [plugins.cri.containerd] + snapshotter = "overlayfs" + no_pivot = false + [plugins.cri.containerd.default_runtime] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "" + runtime_root = "" + [plugins.cri.containerd.untrusted_workload_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + [plugins.cri.cni] + bin_dir = "{{ bin_dir }}" + conf_dir = "/etc/cni/net.d" + conf_template = "" + [plugins.cri.registry] + [plugins.cri.registry.mirrors] + [plugins.cri.registry.mirrors."docker.io"] + endpoint = [ +{% for SITE in REG_MIRRORS %} + "{{ SITE }}", +{% endfor %} + "http://hub-mirror.c.163.com" + ] + [plugins.cri.x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + [plugins.diff-service] + default = ["walking"] + [plugins.linux] + shim = "containerd-shim" + runtime = "runc" + runtime_root = "" + no_shim = false + shim_debug = false + [plugins.opt] + path = "/opt/containerd" + [plugins.restart] + interval = "10s" + [plugins.scheduler] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" diff --git a/roles/containerd/templates/containerd.service.j2 b/roles/containerd/templates/containerd.service.j2 new file mode 100644 index 0000000..65e2169 --- /dev/null +++ b/roles/containerd/templates/containerd.service.j2 @@ -0,0 +1,21 @@ +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target + +[Service] +Environment="PATH={{ bin_dir }}:/bin:/sbin:/usr/bin:/usr/sbin" +ExecStart={{ bin_dir }}/containerd +Restart=always +RestartSec=5 +Delegate=yes +KillMode=process +OOMScoreAdjust=-999 +LimitNOFILE=1048576 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity + +[Install] +WantedBy=multi-user.target diff --git a/roles/containerd/templates/crictl.yaml.j2 b/roles/containerd/templates/crictl.yaml.j2 new file mode 100644 index 0000000..8bfa215 --- /dev/null +++ b/roles/containerd/templates/crictl.yaml.j2 @@ -0,0 +1 @@ +runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/roles/kube-node/templates/kubelet.service.j2 b/roles/kube-node/templates/kubelet.service.j2 index a7d39b1..e6e69c1 100644 --- a/roles/kube-node/templates/kubelet.service.j2 +++ b/roles/kube-node/templates/kubelet.service.j2 @@ -1,8 +1,6 @@ [Unit] Description=Kubernetes Kubelet Documentation=https://github.com/GoogleCloudPlatform/kubernetes -After=docker.service -Requires=docker.service [Service] WorkingDirectory=/var/lib/kubelet @@ -21,6 +19,10 @@ ExecStart={{ bin_dir }}/kubelet \ --cluster-domain={{ CLUSTER_DNS_DOMAIN }} \ --cni-bin-dir={{ bin_dir }} \ --cni-conf-dir=/etc/cni/net.d \ +{% if CONTAINER_RUNTIME == "containerd" %} + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ +{% endif %} --fail-swap-on=false \ --hairpin-mode hairpin-veth \ --hostname-override={{ inventory_hostname }} \ diff --git a/tools/20.addnode.yml b/tools/20.addnode.yml index 3ca6e75..3c677a8 100644 --- a/tools/20.addnode.yml +++ b/tools/20.addnode.yml @@ -5,7 +5,8 @@ roles: - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes'" } - prepare - - docker + - { role: docker, when: "CONTAINER_RUNTIME == 'docker'" } + - { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" } - kube-node - { role: calico, when: "CLUSTER_NETWORK == 'calico'" } - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" } diff --git a/tools/21.addmaster.yml b/tools/21.addmaster.yml index 2c1eff4..b5f3a94 100644 --- a/tools/21.addmaster.yml +++ b/tools/21.addmaster.yml @@ -5,7 +5,8 @@ roles: - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes'" } - prepare - - docker + - { role: docker, when: "CONTAINER_RUNTIME == 'docker'" } + - { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" } - kube-master - kube-node - { role: calico, when: "CLUSTER_NETWORK == 'calico'" } diff --git a/tools/change_ip_aio.yml b/tools/change_ip_aio.yml index 11a0be3..41141ac 100644 --- a/tools/change_ip_aio.yml +++ b/tools/change_ip_aio.yml @@ -2,10 +2,10 @@ # 此脚本仅用于allinone部署情况下，需要修改host ip地址使用（比如，虚机装完allinone部署，克隆或者复制分享给别人测试使用） # # ------- 前提 ：一个运行正常的allinone部署在虚机，关机后复制给别人使用，新虚机开机后如果需要修改IP，请执行如下步骤 -# 0. 拉取最新项目代码：git pull origin master # 1. 修改ansible hosts文件：sed -i 's/$OLD_IP/$NEW_IP/g' /etc/ansible/hosts # 2. 配置ssh免密码登陆：ssh-copy-id $NEW_IP 按提示完成 # 3. 检查下修改是否成功，并且能够成功执行 ansible all -m ping +# 4. 运行本脚本 ansible-playbook /etc/ansible/tools/change_ip_aio.yml - hosts: deploy # hosts 角色无所谓，反正allinone所有角色都是同个ip tasks: