[docker] add support for cri-dockerd as a replacement for dockershim (#8623)
parent
5043517cfb
commit
394857b5ce
|
@ -102,7 +102,6 @@ ENV/
|
|||
|
||||
# molecule
|
||||
roles/**/molecule/**/__pycache__/
|
||||
roles/**/molecule/**/*.conf
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
|
|
@ -37,6 +37,7 @@
|
|||
* [Amazon Linux 2](docs/amazonlinux.md)
|
||||
* CRI
|
||||
* [Containerd](docs/containerd.md)
|
||||
* [Docker](docs/docker.md)
|
||||
* [CRI-O](docs/cri-o.md)
|
||||
* [Kata Containers](docs/kata-containers.md)
|
||||
* [gVisor](docs/gvisor.md)
|
||||
|
|
|
@ -0,0 +1,105 @@
|
|||
# Docker support
|
||||
|
||||
The docker runtime is supported by kubespray and while the `dockershim` is deprecated to be removed in kubernetes 1.24+ there are alternative ways to use docker such as through the [cri-dockerd](https://github.com/Mirantis/cri-dockerd) project supported by Mirantis.
|
||||
|
||||
Using the docker container manager:
|
||||
|
||||
```yaml
|
||||
container_manager: docker
|
||||
```
|
||||
|
||||
Using `cri-dockerd` instead of `dockershim`:
|
||||
|
||||
```yaml
|
||||
cri_dockerd_enabled: false
|
||||
```
|
||||
|
||||
*Note:* The `cri_dockerd_enabled: true` setting will become the default in a future kubespray release once kubespray 1.24+ is supported and `dockershim` is removed. At that point, changing this option will be deprecated and silently ignored.
|
||||
|
||||
Enabling the `overlay2` graph driver:
|
||||
|
||||
```yaml
|
||||
docker_storage_options: -s overlay2
|
||||
```
|
||||
|
||||
Enabling `docker_container_storage_setup`, it will configure devicemapper driver on Centos7 or RedHat7.
|
||||
Deployers must be define a disk path for `docker_container_storage_setup_devs`, otherwise docker-storage-setup will be executed incorrectly.
|
||||
|
||||
```yaml
|
||||
docker_container_storage_setup: true
|
||||
docker_container_storage_setup_devs: /dev/vdb
|
||||
```
|
||||
|
||||
Changing the Docker cgroup driver (native.cgroupdriver); valid options are `systemd` or `cgroupfs`, default is `systemd`:
|
||||
|
||||
```yaml
|
||||
docker_cgroup_driver: systemd
|
||||
```
|
||||
|
||||
If you have more than 3 nameservers kubespray will only use the first 3 else it will fail. Set the `docker_dns_servers_strict` to `false` to prevent deployment failure.
|
||||
|
||||
```yaml
|
||||
docker_dns_servers_strict: false
|
||||
```
|
||||
|
||||
Set the path used to store Docker data:
|
||||
|
||||
```yaml
|
||||
docker_daemon_graph: "/var/lib/docker"
|
||||
```
|
||||
|
||||
Changing the docker daemon iptables support:
|
||||
|
||||
```yaml
|
||||
docker_iptables_enabled: "false"
|
||||
```
|
||||
|
||||
Docker log options:
|
||||
|
||||
```yaml
|
||||
# Rotate container stderr/stdout logs at 50m and keep last 5
|
||||
docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5"
|
||||
```
|
||||
|
||||
Changre the docker `bin_dir`, this should not be changed unless you use a custom docker package:
|
||||
|
||||
```yaml
|
||||
docker_bin_dir: "/usr/bin"
|
||||
```
|
||||
|
||||
To keep docker packages after installation; speeds up repeated ansible provisioning runs when '1'.
|
||||
kubespray deletes the docker package on each run, so caching the package makes sense:
|
||||
|
||||
```yaml
|
||||
docker_rpm_keepcache: 1
|
||||
```
|
||||
|
||||
Allowing insecure-registry access to self hosted registries. Can be ipaddress and domain_name.
|
||||
|
||||
```yaml
|
||||
## example define 172.19.16.11 or mirror.registry.io
|
||||
docker_insecure_registries:
|
||||
- mirror.registry.io
|
||||
- 172.19.16.11
|
||||
```
|
||||
|
||||
Adding other registry, i.e. China registry mirror:
|
||||
|
||||
```yaml
|
||||
docker_registry_mirrors:
|
||||
- https://registry.docker-cn.com
|
||||
- https://mirror.aliyuncs.com
|
||||
```
|
||||
|
||||
Overriding default system MountFlags value. This option takes a mount propagation flag: `shared`, `slave` or `private`, which control whether mounts in the file system namespace set up for docker will receive or propagate mounts and unmounts. Leave empty for system default:
|
||||
|
||||
```yaml
|
||||
docker_mount_flags:
|
||||
```
|
||||
|
||||
Adding extra options to pass to the docker daemon:
|
||||
|
||||
```yaml
|
||||
## This string should be exactly as you wish it to appear.
|
||||
docker_options: ""
|
||||
```
|
|
@ -57,3 +57,6 @@ docker_rpm_keepcache: 1
|
|||
## A string of extra options to pass to the docker daemon.
|
||||
## This string should be exactly as you wish it to appear.
|
||||
# docker_options: ""
|
||||
|
||||
## Use CRI-DockerD instead of dockershim
|
||||
# cri_dockerd_enabled: false
|
||||
|
|
|
@ -0,0 +1,21 @@
|
|||
---
|
||||
- name: restart cri-dockerd
|
||||
command: /bin/true
|
||||
notify:
|
||||
- cri-dockerd | reload systemd
|
||||
- cri-dockerd | reload cri-dockerd.socket
|
||||
- cri-dockerd | reload cri-dockerd.service
|
||||
|
||||
- name: cri-dockerd | reload systemd
|
||||
systemd:
|
||||
daemon_reload: true
|
||||
|
||||
- name: cri-dockerd | reload cri-dockerd.socket
|
||||
service:
|
||||
name: cri-dockerd.socket
|
||||
state: restarted
|
||||
|
||||
- name: cri-dockerd | reload cri-dockerd.service
|
||||
service:
|
||||
name: cri-dockerd.service
|
||||
state: restarted
|
|
@ -0,0 +1,4 @@
|
|||
---
|
||||
dependencies:
|
||||
- role: container-engine/docker
|
||||
- role: container-engine/crictl
|
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
- name: Converge
|
||||
hosts: all
|
||||
become: true
|
||||
vars:
|
||||
container_manager: docker
|
||||
cri_dockerd_enabled: true
|
||||
roles:
|
||||
- role: kubespray-defaults
|
||||
- role: container-engine/cri-dockerd
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"cniVersion": "0.2.0",
|
||||
"name": "mynet",
|
||||
"type": "bridge",
|
||||
"bridge": "cni0",
|
||||
"isGateway": true,
|
||||
"ipMasq": true,
|
||||
"ipam": {
|
||||
"type": "host-local",
|
||||
"subnet": "172.19.0.0/24",
|
||||
"routes": [
|
||||
{
|
||||
"dst": "0.0.0.0/0"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"metadata": {
|
||||
"name": "cri-dockerd1"
|
||||
},
|
||||
"image": {
|
||||
"image": "quay.io/kubespray/hello-world:latest"
|
||||
},
|
||||
"log_path": "cri-dockerd1.0.log",
|
||||
"linux": {}
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"metadata": {
|
||||
"name": "cri-dockerd1",
|
||||
"namespace": "default",
|
||||
"attempt": 1,
|
||||
"uid": "hdishd83djaidwnduwk28bcsb"
|
||||
},
|
||||
"linux": {},
|
||||
"log_directory": "/tmp"
|
||||
}
|
|
@ -0,0 +1,45 @@
|
|||
---
|
||||
driver:
|
||||
name: vagrant
|
||||
provider:
|
||||
name: libvirt
|
||||
options:
|
||||
driver: kvm
|
||||
lint: |
|
||||
set -e
|
||||
yamllint -c ../../../.yamllint .
|
||||
platforms:
|
||||
- name: almalinux8
|
||||
box: almalinux/8
|
||||
cpus: 1
|
||||
memory: 1024
|
||||
nested: true
|
||||
groups:
|
||||
- kube_control_plane
|
||||
- name: ubuntu20
|
||||
box: generic/ubuntu2004
|
||||
cpus: 1
|
||||
memory: 1024
|
||||
nested: true
|
||||
groups:
|
||||
- kube_control_plane
|
||||
provisioner:
|
||||
name: ansible
|
||||
env:
|
||||
ANSIBLE_ROLES_PATH: ../../../../
|
||||
config_options:
|
||||
defaults:
|
||||
callback_whitelist: profile_tasks
|
||||
timeout: 120
|
||||
lint:
|
||||
name: ansible-lint
|
||||
options:
|
||||
c: ../../../.ansible-lint
|
||||
inventory:
|
||||
group_vars:
|
||||
all:
|
||||
become: true
|
||||
verifier:
|
||||
name: testinfra
|
||||
lint:
|
||||
name: flake8
|
|
@ -0,0 +1,47 @@
|
|||
---
|
||||
- name: Prepare
|
||||
hosts: all
|
||||
become: true
|
||||
roles:
|
||||
- role: kubespray-defaults
|
||||
- role: bootstrap-os
|
||||
- role: adduser
|
||||
user: "{{ addusers.kube }}"
|
||||
tasks:
|
||||
- include_tasks: "../../../../download/tasks/download_file.yml"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(downloads.cni) }}"
|
||||
|
||||
- name: Prepare container runtime
|
||||
hosts: all
|
||||
become: true
|
||||
vars:
|
||||
container_manager: containerd
|
||||
kube_network_plugin: cni
|
||||
roles:
|
||||
- role: kubespray-defaults
|
||||
- role: network_plugin/cni
|
||||
tasks:
|
||||
- name: Copy test container files
|
||||
copy:
|
||||
src: "{{ item }}"
|
||||
dest: "/tmp/{{ item }}"
|
||||
owner: root
|
||||
mode: 0644
|
||||
with_items:
|
||||
- container.json
|
||||
- sandbox.json
|
||||
- name: Create /etc/cni/net.d directory
|
||||
file:
|
||||
path: /etc/cni/net.d
|
||||
state: directory
|
||||
owner: kube
|
||||
mode: 0755
|
||||
- name: Setup CNI
|
||||
copy:
|
||||
src: "{{ item }}"
|
||||
dest: "/etc/cni/net.d/{{ item }}"
|
||||
owner: root
|
||||
mode: 0644
|
||||
with_items:
|
||||
- 10-mynet.conf
|
|
@ -0,0 +1,19 @@
|
|||
import os
|
||||
|
||||
import testinfra.utils.ansible_runner
|
||||
|
||||
testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
|
||||
os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
|
||||
|
||||
|
||||
def test_run_pod(host):
|
||||
run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json"
|
||||
with host.sudo():
|
||||
cmd = host.command(run_command)
|
||||
assert cmd.rc == 0
|
||||
|
||||
with host.sudo():
|
||||
log_f = host.file("/tmp/cri-dockerd1.0.log")
|
||||
|
||||
assert log_f.exists
|
||||
assert b"Hello from Docker" in log_f.content
|
|
@ -0,0 +1,25 @@
|
|||
---
|
||||
- name: runc | Download cri-dockerd binary
|
||||
include_tasks: "../../../download/tasks/download_file.yml"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(downloads.cri_dockerd) }}"
|
||||
|
||||
- name: Copy cri-dockerd binary from download dir
|
||||
copy:
|
||||
src: "{{ local_release_dir }}/cri-dockerd"
|
||||
dest: "{{ bin_dir }}/cri-dockerd"
|
||||
mode: 0755
|
||||
remote_src: true
|
||||
notify:
|
||||
- restart cri-dockerd
|
||||
|
||||
- name: Generate cri-dockerd systemd unit files
|
||||
template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
mode: 0644
|
||||
with_items:
|
||||
- cri-dockerd.service
|
||||
- cri-dockerd.socket
|
||||
notify:
|
||||
- restart cri-dockerd
|
|
@ -0,0 +1,39 @@
|
|||
[Unit]
|
||||
Description=CRI Interface for Docker Application Container Engine
|
||||
Documentation=https://docs.mirantis.com
|
||||
After=network-online.target firewalld.service docker.service
|
||||
Wants=network-online.target docker.service
|
||||
Requires=cri-dockerd.socket
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint fd:// --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet }}
|
||||
ExecReload=/bin/kill -s HUP $MAINPID
|
||||
TimeoutSec=0
|
||||
RestartSec=2
|
||||
Restart=always
|
||||
|
||||
# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
|
||||
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
|
||||
# to make them work for either version of systemd.
|
||||
StartLimitBurst=3
|
||||
|
||||
# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
|
||||
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
|
||||
# this option work for either version of systemd.
|
||||
StartLimitInterval=60s
|
||||
|
||||
# Having non-zero Limit*s causes performance problems due to accounting overhead
|
||||
# in the kernel. We recommend using cgroups to do container-local accounting.
|
||||
LimitNOFILE=infinity
|
||||
LimitNPROC=infinity
|
||||
LimitCORE=infinity
|
||||
|
||||
# Comment TasksMax if your systemd version does not support it.
|
||||
# Only systemd 226 and above support this option.
|
||||
TasksMax=infinity
|
||||
Delegate=yes
|
||||
KillMode=process
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -0,0 +1,12 @@
|
|||
[Unit]
|
||||
Description=CRI Docker Socket for the API
|
||||
PartOf=cri-dockerd.service
|
||||
|
||||
[Socket]
|
||||
ListenStream=%t/cri-dockerd.sock
|
||||
SocketMode=0660
|
||||
SocketUser=root
|
||||
SocketGroup=docker
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
|
@ -56,3 +56,11 @@ dependencies:
|
|||
tags:
|
||||
- container-engine
|
||||
- docker
|
||||
|
||||
- role: container-engine/cri-dockerd
|
||||
when:
|
||||
- cri_dockerd_enabled
|
||||
- container_manager == 'docker'
|
||||
tags:
|
||||
- container-engine
|
||||
- docker
|
||||
|
|
|
@ -75,6 +75,7 @@ kata_containers_version: 2.2.3
|
|||
youki_version: 0.0.1
|
||||
gvisor_version: 20210921
|
||||
containerd_version: 1.6.1
|
||||
cri_dockerd_version: v0.2.0
|
||||
|
||||
# this is relevant when container_manager == 'docker'
|
||||
docker_containerd_version: 1.4.12
|
||||
|
@ -152,6 +153,7 @@ gvisor_containerd_shim_runsc_download_url: "https://storage.googleapis.com/gviso
|
|||
nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
|
||||
krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz"
|
||||
containerd_download_url: "https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
|
||||
cri_dockerd_download_url: "https://github.com/Mirantis/cri-dockerd/releases/download/{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}-linux-{{ image_arch }}.tar.gz"
|
||||
|
||||
crictl_checksums:
|
||||
arm:
|
||||
|
@ -577,6 +579,16 @@ helm_archive_checksums:
|
|||
ppc64le:
|
||||
v3.8.0: 5070fa5188e7bc798dd54bc1ea9fc4cda623d9ff45eedb05ec93db234309f391
|
||||
|
||||
cri_dockerd_archive_checksums:
|
||||
arm:
|
||||
v0.2.0: 0
|
||||
arm64:
|
||||
v0.2.0: 6ddcb77bc7ca9d93008fdcb81659e3acfb45105dc935782030afb7dab4a3250b
|
||||
amd64:
|
||||
v0.2.0: ad0e5f2ac6e7a57af4b560d1638ee26baab68c56a49ff3337e10ff9cea0a3539
|
||||
ppc64le:
|
||||
v0.2.0: 0
|
||||
|
||||
runc_checksums:
|
||||
arm:
|
||||
v1.0.2: 0
|
||||
|
@ -738,6 +750,7 @@ kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}"
|
|||
calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
|
||||
calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}"
|
||||
crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
|
||||
cri_dockerd_archive_checksum: "{{ cri_dockerd_archive_checksums[image_arch][cri_dockerd_version] }}"
|
||||
helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}"
|
||||
runc_binary_checksum: "{{ runc_checksums[image_arch][runc_version] }}"
|
||||
crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}"
|
||||
|
@ -1011,7 +1024,7 @@ downloads:
|
|||
|
||||
crictl:
|
||||
file: true
|
||||
enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] }}"
|
||||
enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] or cri_dockerd_enabled }}"
|
||||
version: "{{ crictl_version }}"
|
||||
dest: "{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz"
|
||||
sha256: "{{ crictl_binary_checksum }}"
|
||||
|
@ -1022,6 +1035,19 @@ downloads:
|
|||
groups:
|
||||
- k8s_cluster
|
||||
|
||||
cri_dockerd:
|
||||
file: true
|
||||
enabled: "{{ cri_dockerd_enabled }}"
|
||||
version: "{{ cri_dockerd_version }}"
|
||||
dest: "{{ local_release_dir }}/cri-dockerd-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz"
|
||||
sha256: "{{ cri_dockerd_archive_checksum }}"
|
||||
url: "{{ cri_dockerd_download_url }}"
|
||||
unarchive: true
|
||||
owner: "root"
|
||||
mode: "0755"
|
||||
groups:
|
||||
- k8s_cluster
|
||||
|
||||
crun:
|
||||
file: true
|
||||
enabled: "{{ crun_enabled }}"
|
||||
|
|
|
@ -12,7 +12,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
|
|||
--config={{ kube_config_dir }}/kubelet-config.yaml \
|
||||
--kubeconfig={{ kube_config_dir }}/kubelet.conf \
|
||||
{# end kubeadm specific settings #}
|
||||
{% if container_manager == 'docker' %}
|
||||
{% if container_manager == 'docker' and not cri_dockerd_enabled %}
|
||||
--pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
|
||||
{% else %}
|
||||
--container-runtime=remote \
|
||||
|
|
|
@ -263,6 +263,9 @@ deploy_container_engine: "{{ inventory_hostname in groups['k8s_cluster'] or etcd
|
|||
# Container for runtime
|
||||
container_manager: containerd
|
||||
|
||||
# Enable CRI Docker interface
|
||||
cri_dockerd_enabled: false
|
||||
|
||||
# Enable Kata Containers as additional container runtime
|
||||
# When enabled, it requires `container_manager` different than Docker
|
||||
kata_containers_enabled: false
|
||||
|
@ -288,6 +291,8 @@ cri_socket: >-
|
|||
/var/run/crio/crio.sock
|
||||
{%- elif container_manager == 'containerd' -%}
|
||||
/var/run/containerd/containerd.sock
|
||||
{%- elif cri_dockerd_enabled -%}
|
||||
/var/run/cri-dockerd.sock
|
||||
{%- else -%}
|
||||
/var/run/dockershim.sock
|
||||
{%- endif -%}
|
||||
|
|
|
@ -4,7 +4,9 @@
|
|||
name: "{{ item }}"
|
||||
state: stopped
|
||||
with_items:
|
||||
- kubelet
|
||||
- kubelet.service
|
||||
- cri-dockerd.service
|
||||
- cri-dockerd.socket
|
||||
failed_when: false
|
||||
tags:
|
||||
- services
|
||||
|
@ -15,6 +17,8 @@
|
|||
state: absent
|
||||
with_items:
|
||||
- kubelet.service
|
||||
- cri-dockerd.service
|
||||
- cri-dockerd.socket
|
||||
- calico-node.service
|
||||
- containerd.service.d/http-proxy.conf
|
||||
- crio.service.d/http-proxy.conf
|
||||
|
@ -299,6 +303,7 @@
|
|||
- /etc/pki/ca-trust/source/anchors/kube-ca.crt
|
||||
- /var/log/pods/
|
||||
- "{{ bin_dir }}/kubelet"
|
||||
- "{{ bin_dir }}/cri-dockerd"
|
||||
- "{{ bin_dir }}/etcd-scripts"
|
||||
- "{{ bin_dir }}/etcd"
|
||||
- "{{ bin_dir }}/etcd-events"
|
||||
|
|
|
@ -8,3 +8,4 @@ vm_memory: 3072Mi
|
|||
container_manager: docker
|
||||
etcd_deployment_type: docker
|
||||
resolvconf_mode: docker_dns
|
||||
cri_dockerd_enabled: true
|
||||
|
|
|
@ -15,3 +15,4 @@ enable_nodelocaldns: False
|
|||
container_manager: docker
|
||||
etcd_deployment_type: docker
|
||||
resolvconf_mode: docker_dns
|
||||
cri_dockerd_enabled: true
|
||||
|
|
Loading…
Reference in New Issue