[docker] add support for cri-dockerd as a replacement for dockershim (#8623)

pull/8629/head
Cristian Calin 2022-03-17 01:28:11 +02:00 committed by GitHub
parent 5043517cfb
commit 394857b5ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 417 additions and 4 deletions

1
.gitignore vendored
View File

@ -102,7 +102,6 @@ ENV/
# molecule # molecule
roles/**/molecule/**/__pycache__/ roles/**/molecule/**/__pycache__/
roles/**/molecule/**/*.conf
# macOS # macOS
.DS_Store .DS_Store

View File

@ -37,6 +37,7 @@
* [Amazon Linux 2](docs/amazonlinux.md) * [Amazon Linux 2](docs/amazonlinux.md)
* CRI * CRI
* [Containerd](docs/containerd.md) * [Containerd](docs/containerd.md)
* [Docker](docs/docker.md)
* [CRI-O](docs/cri-o.md) * [CRI-O](docs/cri-o.md)
* [Kata Containers](docs/kata-containers.md) * [Kata Containers](docs/kata-containers.md)
* [gVisor](docs/gvisor.md) * [gVisor](docs/gvisor.md)

105
docs/docker.md 100644
View File

@ -0,0 +1,105 @@
# Docker support
The docker runtime is supported by kubespray and while the `dockershim` is deprecated to be removed in kubernetes 1.24+ there are alternative ways to use docker such as through the [cri-dockerd](https://github.com/Mirantis/cri-dockerd) project supported by Mirantis.
Using the docker container manager:
```yaml
container_manager: docker
```
Using `cri-dockerd` instead of `dockershim`:
```yaml
cri_dockerd_enabled: false
```
*Note:* The `cri_dockerd_enabled: true` setting will become the default in a future kubespray release once kubespray 1.24+ is supported and `dockershim` is removed. At that point, changing this option will be deprecated and silently ignored.
Enabling the `overlay2` graph driver:
```yaml
docker_storage_options: -s overlay2
```
Enabling `docker_container_storage_setup`, it will configure devicemapper driver on Centos7 or RedHat7.
Deployers must be define a disk path for `docker_container_storage_setup_devs`, otherwise docker-storage-setup will be executed incorrectly.
```yaml
docker_container_storage_setup: true
docker_container_storage_setup_devs: /dev/vdb
```
Changing the Docker cgroup driver (native.cgroupdriver); valid options are `systemd` or `cgroupfs`, default is `systemd`:
```yaml
docker_cgroup_driver: systemd
```
If you have more than 3 nameservers kubespray will only use the first 3 else it will fail. Set the `docker_dns_servers_strict` to `false` to prevent deployment failure.
```yaml
docker_dns_servers_strict: false
```
Set the path used to store Docker data:
```yaml
docker_daemon_graph: "/var/lib/docker"
```
Changing the docker daemon iptables support:
```yaml
docker_iptables_enabled: "false"
```
Docker log options:
```yaml
# Rotate container stderr/stdout logs at 50m and keep last 5
docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5"
```
Changre the docker `bin_dir`, this should not be changed unless you use a custom docker package:
```yaml
docker_bin_dir: "/usr/bin"
```
To keep docker packages after installation; speeds up repeated ansible provisioning runs when '1'.
kubespray deletes the docker package on each run, so caching the package makes sense:
```yaml
docker_rpm_keepcache: 1
```
Allowing insecure-registry access to self hosted registries. Can be ipaddress and domain_name.
```yaml
## example define 172.19.16.11 or mirror.registry.io
docker_insecure_registries:
- mirror.registry.io
- 172.19.16.11
```
Adding other registry, i.e. China registry mirror:
```yaml
docker_registry_mirrors:
- https://registry.docker-cn.com
- https://mirror.aliyuncs.com
```
Overriding default system MountFlags value. This option takes a mount propagation flag: `shared`, `slave` or `private`, which control whether mounts in the file system namespace set up for docker will receive or propagate mounts and unmounts. Leave empty for system default:
```yaml
docker_mount_flags:
```
Adding extra options to pass to the docker daemon:
```yaml
## This string should be exactly as you wish it to appear.
docker_options: ""
```

View File

@ -57,3 +57,6 @@ docker_rpm_keepcache: 1
## A string of extra options to pass to the docker daemon. ## A string of extra options to pass to the docker daemon.
## This string should be exactly as you wish it to appear. ## This string should be exactly as you wish it to appear.
# docker_options: "" # docker_options: ""
## Use CRI-DockerD instead of dockershim
# cri_dockerd_enabled: false

View File

@ -0,0 +1,21 @@
---
- name: restart cri-dockerd
command: /bin/true
notify:
- cri-dockerd | reload systemd
- cri-dockerd | reload cri-dockerd.socket
- cri-dockerd | reload cri-dockerd.service
- name: cri-dockerd | reload systemd
systemd:
daemon_reload: true
- name: cri-dockerd | reload cri-dockerd.socket
service:
name: cri-dockerd.socket
state: restarted
- name: cri-dockerd | reload cri-dockerd.service
service:
name: cri-dockerd.service
state: restarted

View File

@ -0,0 +1,4 @@
---
dependencies:
- role: container-engine/docker
- role: container-engine/crictl

View File

@ -0,0 +1,10 @@
---
- name: Converge
hosts: all
become: true
vars:
container_manager: docker
cri_dockerd_enabled: true
roles:
- role: kubespray-defaults
- role: container-engine/cri-dockerd

View File

@ -0,0 +1,17 @@
{
"cniVersion": "0.2.0",
"name": "mynet",
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "172.19.0.0/24",
"routes": [
{
"dst": "0.0.0.0/0"
}
]
}
}

View File

@ -0,0 +1,10 @@
{
"metadata": {
"name": "cri-dockerd1"
},
"image": {
"image": "quay.io/kubespray/hello-world:latest"
},
"log_path": "cri-dockerd1.0.log",
"linux": {}
}

View File

@ -0,0 +1,10 @@
{
"metadata": {
"name": "cri-dockerd1",
"namespace": "default",
"attempt": 1,
"uid": "hdishd83djaidwnduwk28bcsb"
},
"linux": {},
"log_directory": "/tmp"
}

View File

@ -0,0 +1,45 @@
---
driver:
name: vagrant
provider:
name: libvirt
options:
driver: kvm
lint: |
set -e
yamllint -c ../../../.yamllint .
platforms:
- name: almalinux8
box: almalinux/8
cpus: 1
memory: 1024
nested: true
groups:
- kube_control_plane
- name: ubuntu20
box: generic/ubuntu2004
cpus: 1
memory: 1024
nested: true
groups:
- kube_control_plane
provisioner:
name: ansible
env:
ANSIBLE_ROLES_PATH: ../../../../
config_options:
defaults:
callback_whitelist: profile_tasks
timeout: 120
lint:
name: ansible-lint
options:
c: ../../../.ansible-lint
inventory:
group_vars:
all:
become: true
verifier:
name: testinfra
lint:
name: flake8

View File

@ -0,0 +1,47 @@
---
- name: Prepare
hosts: all
become: true
roles:
- role: kubespray-defaults
- role: bootstrap-os
- role: adduser
user: "{{ addusers.kube }}"
tasks:
- include_tasks: "../../../../download/tasks/download_file.yml"
vars:
download: "{{ download_defaults | combine(downloads.cni) }}"
- name: Prepare container runtime
hosts: all
become: true
vars:
container_manager: containerd
kube_network_plugin: cni
roles:
- role: kubespray-defaults
- role: network_plugin/cni
tasks:
- name: Copy test container files
copy:
src: "{{ item }}"
dest: "/tmp/{{ item }}"
owner: root
mode: 0644
with_items:
- container.json
- sandbox.json
- name: Create /etc/cni/net.d directory
file:
path: /etc/cni/net.d
state: directory
owner: kube
mode: 0755
- name: Setup CNI
copy:
src: "{{ item }}"
dest: "/etc/cni/net.d/{{ item }}"
owner: root
mode: 0644
with_items:
- 10-mynet.conf

View File

@ -0,0 +1,19 @@
import os
import testinfra.utils.ansible_runner
testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
def test_run_pod(host):
run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json"
with host.sudo():
cmd = host.command(run_command)
assert cmd.rc == 0
with host.sudo():
log_f = host.file("/tmp/cri-dockerd1.0.log")
assert log_f.exists
assert b"Hello from Docker" in log_f.content

View File

@ -0,0 +1,25 @@
---
- name: runc | Download cri-dockerd binary
include_tasks: "../../../download/tasks/download_file.yml"
vars:
download: "{{ download_defaults | combine(downloads.cri_dockerd) }}"
- name: Copy cri-dockerd binary from download dir
copy:
src: "{{ local_release_dir }}/cri-dockerd"
dest: "{{ bin_dir }}/cri-dockerd"
mode: 0755
remote_src: true
notify:
- restart cri-dockerd
- name: Generate cri-dockerd systemd unit files
template:
src: "{{ item }}.j2"
dest: "/etc/systemd/system/{{ item }}"
mode: 0644
with_items:
- cri-dockerd.service
- cri-dockerd.socket
notify:
- restart cri-dockerd

View File

@ -0,0 +1,39 @@
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target docker.service
Requires=cri-dockerd.socket
[Service]
Type=notify
ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint fd:// --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet }}
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3
# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,12 @@
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-dockerd.service
[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker
[Install]
WantedBy=sockets.target

View File

@ -56,3 +56,11 @@ dependencies:
tags: tags:
- container-engine - container-engine
- docker - docker
- role: container-engine/cri-dockerd
when:
- cri_dockerd_enabled
- container_manager == 'docker'
tags:
- container-engine
- docker

View File

@ -75,6 +75,7 @@ kata_containers_version: 2.2.3
youki_version: 0.0.1 youki_version: 0.0.1
gvisor_version: 20210921 gvisor_version: 20210921
containerd_version: 1.6.1 containerd_version: 1.6.1
cri_dockerd_version: v0.2.0
# this is relevant when container_manager == 'docker' # this is relevant when container_manager == 'docker'
docker_containerd_version: 1.4.12 docker_containerd_version: 1.4.12
@ -152,6 +153,7 @@ gvisor_containerd_shim_runsc_download_url: "https://storage.googleapis.com/gviso
nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz" krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz"
containerd_download_url: "https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz" containerd_download_url: "https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
cri_dockerd_download_url: "https://github.com/Mirantis/cri-dockerd/releases/download/{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}-linux-{{ image_arch }}.tar.gz"
crictl_checksums: crictl_checksums:
arm: arm:
@ -577,6 +579,16 @@ helm_archive_checksums:
ppc64le: ppc64le:
v3.8.0: 5070fa5188e7bc798dd54bc1ea9fc4cda623d9ff45eedb05ec93db234309f391 v3.8.0: 5070fa5188e7bc798dd54bc1ea9fc4cda623d9ff45eedb05ec93db234309f391
cri_dockerd_archive_checksums:
arm:
v0.2.0: 0
arm64:
v0.2.0: 6ddcb77bc7ca9d93008fdcb81659e3acfb45105dc935782030afb7dab4a3250b
amd64:
v0.2.0: ad0e5f2ac6e7a57af4b560d1638ee26baab68c56a49ff3337e10ff9cea0a3539
ppc64le:
v0.2.0: 0
runc_checksums: runc_checksums:
arm: arm:
v1.0.2: 0 v1.0.2: 0
@ -738,6 +750,7 @@ kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}"
calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}" calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}" calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}"
crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
cri_dockerd_archive_checksum: "{{ cri_dockerd_archive_checksums[image_arch][cri_dockerd_version] }}"
helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}" helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}"
runc_binary_checksum: "{{ runc_checksums[image_arch][runc_version] }}" runc_binary_checksum: "{{ runc_checksums[image_arch][runc_version] }}"
crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}" crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}"
@ -1011,7 +1024,7 @@ downloads:
crictl: crictl:
file: true file: true
enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] }}" enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] or cri_dockerd_enabled }}"
version: "{{ crictl_version }}" version: "{{ crictl_version }}"
dest: "{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz" dest: "{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz"
sha256: "{{ crictl_binary_checksum }}" sha256: "{{ crictl_binary_checksum }}"
@ -1022,6 +1035,19 @@ downloads:
groups: groups:
- k8s_cluster - k8s_cluster
cri_dockerd:
file: true
enabled: "{{ cri_dockerd_enabled }}"
version: "{{ cri_dockerd_version }}"
dest: "{{ local_release_dir }}/cri-dockerd-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz"
sha256: "{{ cri_dockerd_archive_checksum }}"
url: "{{ cri_dockerd_download_url }}"
unarchive: true
owner: "root"
mode: "0755"
groups:
- k8s_cluster
crun: crun:
file: true file: true
enabled: "{{ crun_enabled }}" enabled: "{{ crun_enabled }}"

View File

@ -12,7 +12,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
--config={{ kube_config_dir }}/kubelet-config.yaml \ --config={{ kube_config_dir }}/kubelet-config.yaml \
--kubeconfig={{ kube_config_dir }}/kubelet.conf \ --kubeconfig={{ kube_config_dir }}/kubelet.conf \
{# end kubeadm specific settings #} {# end kubeadm specific settings #}
{% if container_manager == 'docker' %} {% if container_manager == 'docker' and not cri_dockerd_enabled %}
--pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
{% else %} {% else %}
--container-runtime=remote \ --container-runtime=remote \

View File

@ -263,6 +263,9 @@ deploy_container_engine: "{{ inventory_hostname in groups['k8s_cluster'] or etcd
# Container for runtime # Container for runtime
container_manager: containerd container_manager: containerd
# Enable CRI Docker interface
cri_dockerd_enabled: false
# Enable Kata Containers as additional container runtime # Enable Kata Containers as additional container runtime
# When enabled, it requires `container_manager` different than Docker # When enabled, it requires `container_manager` different than Docker
kata_containers_enabled: false kata_containers_enabled: false
@ -288,6 +291,8 @@ cri_socket: >-
/var/run/crio/crio.sock /var/run/crio/crio.sock
{%- elif container_manager == 'containerd' -%} {%- elif container_manager == 'containerd' -%}
/var/run/containerd/containerd.sock /var/run/containerd/containerd.sock
{%- elif cri_dockerd_enabled -%}
/var/run/cri-dockerd.sock
{%- else -%} {%- else -%}
/var/run/dockershim.sock /var/run/dockershim.sock
{%- endif -%} {%- endif -%}

View File

@ -4,7 +4,9 @@
name: "{{ item }}" name: "{{ item }}"
state: stopped state: stopped
with_items: with_items:
- kubelet - kubelet.service
- cri-dockerd.service
- cri-dockerd.socket
failed_when: false failed_when: false
tags: tags:
- services - services
@ -15,6 +17,8 @@
state: absent state: absent
with_items: with_items:
- kubelet.service - kubelet.service
- cri-dockerd.service
- cri-dockerd.socket
- calico-node.service - calico-node.service
- containerd.service.d/http-proxy.conf - containerd.service.d/http-proxy.conf
- crio.service.d/http-proxy.conf - crio.service.d/http-proxy.conf
@ -299,6 +303,7 @@
- /etc/pki/ca-trust/source/anchors/kube-ca.crt - /etc/pki/ca-trust/source/anchors/kube-ca.crt
- /var/log/pods/ - /var/log/pods/
- "{{ bin_dir }}/kubelet" - "{{ bin_dir }}/kubelet"
- "{{ bin_dir }}/cri-dockerd"
- "{{ bin_dir }}/etcd-scripts" - "{{ bin_dir }}/etcd-scripts"
- "{{ bin_dir }}/etcd" - "{{ bin_dir }}/etcd"
- "{{ bin_dir }}/etcd-events" - "{{ bin_dir }}/etcd-events"

View File

@ -8,3 +8,4 @@ vm_memory: 3072Mi
container_manager: docker container_manager: docker
etcd_deployment_type: docker etcd_deployment_type: docker
resolvconf_mode: docker_dns resolvconf_mode: docker_dns
cri_dockerd_enabled: true

View File

@ -15,3 +15,4 @@ enable_nodelocaldns: False
container_manager: docker container_manager: docker
etcd_deployment_type: docker etcd_deployment_type: docker
resolvconf_mode: docker_dns resolvconf_mode: docker_dns
cri_dockerd_enabled: true