diff --git a/.gitignore b/.gitignore index e0a47ac6f..9856fd0db 100644 --- a/.gitignore +++ b/.gitignore @@ -102,7 +102,6 @@ ENV/ # molecule roles/**/molecule/**/__pycache__/ -roles/**/molecule/**/*.conf # macOS .DS_Store diff --git a/docs/_sidebar.md b/docs/_sidebar.md index b2ca1c51d..0a754206b 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -37,6 +37,7 @@ * [Amazon Linux 2](docs/amazonlinux.md) * CRI * [Containerd](docs/containerd.md) + * [Docker](docs/docker.md) * [CRI-O](docs/cri-o.md) * [Kata Containers](docs/kata-containers.md) * [gVisor](docs/gvisor.md) diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 000000000..e68d55fa0 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,105 @@ +# Docker support + +The docker runtime is supported by kubespray and while the `dockershim` is deprecated to be removed in kubernetes 1.24+ there are alternative ways to use docker such as through the [cri-dockerd](https://github.com/Mirantis/cri-dockerd) project supported by Mirantis. + +Using the docker container manager: + +```yaml +container_manager: docker +``` + +Using `cri-dockerd` instead of `dockershim`: + +```yaml +cri_dockerd_enabled: false +``` + +*Note:* The `cri_dockerd_enabled: true` setting will become the default in a future kubespray release once kubespray 1.24+ is supported and `dockershim` is removed. At that point, changing this option will be deprecated and silently ignored. + +Enabling the `overlay2` graph driver: + +```yaml +docker_storage_options: -s overlay2 +``` + +Enabling `docker_container_storage_setup`, it will configure devicemapper driver on Centos7 or RedHat7. +Deployers must be define a disk path for `docker_container_storage_setup_devs`, otherwise docker-storage-setup will be executed incorrectly. + +```yaml +docker_container_storage_setup: true +docker_container_storage_setup_devs: /dev/vdb +``` + +Changing the Docker cgroup driver (native.cgroupdriver); valid options are `systemd` or `cgroupfs`, default is `systemd`: + +```yaml +docker_cgroup_driver: systemd +``` + +If you have more than 3 nameservers kubespray will only use the first 3 else it will fail. Set the `docker_dns_servers_strict` to `false` to prevent deployment failure. + +```yaml +docker_dns_servers_strict: false +``` + +Set the path used to store Docker data: + +```yaml +docker_daemon_graph: "/var/lib/docker" +``` + +Changing the docker daemon iptables support: + +```yaml +docker_iptables_enabled: "false" +``` + +Docker log options: + +```yaml +# Rotate container stderr/stdout logs at 50m and keep last 5 +docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" +``` + +Changre the docker `bin_dir`, this should not be changed unless you use a custom docker package: + +```yaml +docker_bin_dir: "/usr/bin" +``` + +To keep docker packages after installation; speeds up repeated ansible provisioning runs when '1'. +kubespray deletes the docker package on each run, so caching the package makes sense: + +```yaml +docker_rpm_keepcache: 1 +``` + +Allowing insecure-registry access to self hosted registries. Can be ipaddress and domain_name. + +```yaml +## example define 172.19.16.11 or mirror.registry.io +docker_insecure_registries: + - mirror.registry.io + - 172.19.16.11 +``` + +Adding other registry, i.e. China registry mirror: + +```yaml +docker_registry_mirrors: + - https://registry.docker-cn.com + - https://mirror.aliyuncs.com +``` + +Overriding default system MountFlags value. This option takes a mount propagation flag: `shared`, `slave` or `private`, which control whether mounts in the file system namespace set up for docker will receive or propagate mounts and unmounts. Leave empty for system default: + +```yaml +docker_mount_flags: +``` + +Adding extra options to pass to the docker daemon: + +```yaml +## This string should be exactly as you wish it to appear. +docker_options: "" +``` diff --git a/inventory/sample/group_vars/all/docker.yml b/inventory/sample/group_vars/all/docker.yml index 4e968c300..f7a958712 100644 --- a/inventory/sample/group_vars/all/docker.yml +++ b/inventory/sample/group_vars/all/docker.yml @@ -57,3 +57,6 @@ docker_rpm_keepcache: 1 ## A string of extra options to pass to the docker daemon. ## This string should be exactly as you wish it to appear. # docker_options: "" + +## Use CRI-DockerD instead of dockershim +# cri_dockerd_enabled: false diff --git a/roles/container-engine/cri-dockerd/handlers/main.yml b/roles/container-engine/cri-dockerd/handlers/main.yml new file mode 100644 index 000000000..94b760a02 --- /dev/null +++ b/roles/container-engine/cri-dockerd/handlers/main.yml @@ -0,0 +1,21 @@ +--- +- name: restart cri-dockerd + command: /bin/true + notify: + - cri-dockerd | reload systemd + - cri-dockerd | reload cri-dockerd.socket + - cri-dockerd | reload cri-dockerd.service + +- name: cri-dockerd | reload systemd + systemd: + daemon_reload: true + +- name: cri-dockerd | reload cri-dockerd.socket + service: + name: cri-dockerd.socket + state: restarted + +- name: cri-dockerd | reload cri-dockerd.service + service: + name: cri-dockerd.service + state: restarted diff --git a/roles/container-engine/cri-dockerd/meta/main.yml b/roles/container-engine/cri-dockerd/meta/main.yml new file mode 100644 index 000000000..4923f3b0b --- /dev/null +++ b/roles/container-engine/cri-dockerd/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: container-engine/docker + - role: container-engine/crictl diff --git a/roles/container-engine/cri-dockerd/molecule/default/converge.yml b/roles/container-engine/cri-dockerd/molecule/default/converge.yml new file mode 100644 index 000000000..66ace6c95 --- /dev/null +++ b/roles/container-engine/cri-dockerd/molecule/default/converge.yml @@ -0,0 +1,10 @@ +--- +- name: Converge + hosts: all + become: true + vars: + container_manager: docker + cri_dockerd_enabled: true + roles: + - role: kubespray-defaults + - role: container-engine/cri-dockerd diff --git a/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf b/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf new file mode 100644 index 000000000..f10935b75 --- /dev/null +++ b/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/roles/container-engine/cri-dockerd/molecule/default/files/container.json b/roles/container-engine/cri-dockerd/molecule/default/files/container.json new file mode 100644 index 000000000..1d839e695 --- /dev/null +++ b/roles/container-engine/cri-dockerd/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "cri-dockerd1" + }, + "image": { + "image": "quay.io/kubespray/hello-world:latest" + }, + "log_path": "cri-dockerd1.0.log", + "linux": {} +} diff --git a/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json b/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json new file mode 100644 index 000000000..f451e9e54 --- /dev/null +++ b/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "cri-dockerd1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/roles/container-engine/cri-dockerd/molecule/default/molecule.yml b/roles/container-engine/cri-dockerd/molecule/default/molecule.yml new file mode 100644 index 000000000..e99c23cc1 --- /dev/null +++ b/roles/container-engine/cri-dockerd/molecule/default/molecule.yml @@ -0,0 +1,45 @@ +--- +driver: + name: vagrant + provider: + name: libvirt + options: + driver: kvm +lint: | + set -e + yamllint -c ../../../.yamllint . +platforms: + - name: almalinux8 + box: almalinux/8 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callback_whitelist: profile_tasks + timeout: 120 + lint: + name: ansible-lint + options: + c: ../../../.ansible-lint + inventory: + group_vars: + all: + become: true +verifier: + name: testinfra + lint: + name: flake8 diff --git a/roles/container-engine/cri-dockerd/molecule/default/prepare.yml b/roles/container-engine/cri-dockerd/molecule/default/prepare.yml new file mode 100644 index 000000000..77e48b083 --- /dev/null +++ b/roles/container-engine/cri-dockerd/molecule/default/prepare.yml @@ -0,0 +1,47 @@ +--- +- name: Prepare + hosts: all + become: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: adduser + user: "{{ addusers.kube }}" + tasks: + - include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare container runtime + hosts: all + become: true + vars: + container_manager: containerd + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: network_plugin/cni + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: kube + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py b/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py new file mode 100644 index 000000000..dc99b3498 --- /dev/null +++ b/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py @@ -0,0 +1,19 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_run_pod(host): + run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json" + with host.sudo(): + cmd = host.command(run_command) + assert cmd.rc == 0 + + with host.sudo(): + log_f = host.file("/tmp/cri-dockerd1.0.log") + + assert log_f.exists + assert b"Hello from Docker" in log_f.content diff --git a/roles/container-engine/cri-dockerd/tasks/main.yml b/roles/container-engine/cri-dockerd/tasks/main.yml new file mode 100644 index 000000000..49ba8ef01 --- /dev/null +++ b/roles/container-engine/cri-dockerd/tasks/main.yml @@ -0,0 +1,25 @@ +--- +- name: runc | Download cri-dockerd binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cri_dockerd) }}" + +- name: Copy cri-dockerd binary from download dir + copy: + src: "{{ local_release_dir }}/cri-dockerd" + dest: "{{ bin_dir }}/cri-dockerd" + mode: 0755 + remote_src: true + notify: + - restart cri-dockerd + +- name: Generate cri-dockerd systemd unit files + template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: 0644 + with_items: + - cri-dockerd.service + - cri-dockerd.socket + notify: + - restart cri-dockerd diff --git a/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 b/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 new file mode 100644 index 000000000..247f5ad4d --- /dev/null +++ b/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 @@ -0,0 +1,39 @@ +[Unit] +Description=CRI Interface for Docker Application Container Engine +Documentation=https://docs.mirantis.com +After=network-online.target firewalld.service docker.service +Wants=network-online.target docker.service +Requires=cri-dockerd.socket + +[Service] +Type=notify +ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint fd:// --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet }} +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not support it. +# Only systemd 226 and above support this option. +TasksMax=infinity +Delegate=yes +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2 b/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2 new file mode 100644 index 000000000..8dfa27d44 --- /dev/null +++ b/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=CRI Docker Socket for the API +PartOf=cri-dockerd.service + +[Socket] +ListenStream=%t/cri-dockerd.sock +SocketMode=0660 +SocketUser=root +SocketGroup=docker + +[Install] +WantedBy=sockets.target diff --git a/roles/container-engine/meta/main.yml b/roles/container-engine/meta/main.yml index bed9c13a2..1b8fce430 100644 --- a/roles/container-engine/meta/main.yml +++ b/roles/container-engine/meta/main.yml @@ -56,3 +56,11 @@ dependencies: tags: - container-engine - docker + + - role: container-engine/cri-dockerd + when: + - cri_dockerd_enabled + - container_manager == 'docker' + tags: + - container-engine + - docker diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 33260ee2f..a8a227624 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -75,6 +75,7 @@ kata_containers_version: 2.2.3 youki_version: 0.0.1 gvisor_version: 20210921 containerd_version: 1.6.1 +cri_dockerd_version: v0.2.0 # this is relevant when container_manager == 'docker' docker_containerd_version: 1.4.12 @@ -152,6 +153,7 @@ gvisor_containerd_shim_runsc_download_url: "https://storage.googleapis.com/gviso nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz" containerd_download_url: "https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz" +cri_dockerd_download_url: "https://github.com/Mirantis/cri-dockerd/releases/download/{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}-linux-{{ image_arch }}.tar.gz" crictl_checksums: arm: @@ -577,6 +579,16 @@ helm_archive_checksums: ppc64le: v3.8.0: 5070fa5188e7bc798dd54bc1ea9fc4cda623d9ff45eedb05ec93db234309f391 +cri_dockerd_archive_checksums: + arm: + v0.2.0: 0 + arm64: + v0.2.0: 6ddcb77bc7ca9d93008fdcb81659e3acfb45105dc935782030afb7dab4a3250b + amd64: + v0.2.0: ad0e5f2ac6e7a57af4b560d1638ee26baab68c56a49ff3337e10ff9cea0a3539 + ppc64le: + v0.2.0: 0 + runc_checksums: arm: v1.0.2: 0 @@ -738,6 +750,7 @@ kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}" calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}" calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}" crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" +cri_dockerd_archive_checksum: "{{ cri_dockerd_archive_checksums[image_arch][cri_dockerd_version] }}" helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}" runc_binary_checksum: "{{ runc_checksums[image_arch][runc_version] }}" crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}" @@ -1011,7 +1024,7 @@ downloads: crictl: file: true - enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] }}" + enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] or cri_dockerd_enabled }}" version: "{{ crictl_version }}" dest: "{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz" sha256: "{{ crictl_binary_checksum }}" @@ -1022,6 +1035,19 @@ downloads: groups: - k8s_cluster + cri_dockerd: + file: true + enabled: "{{ cri_dockerd_enabled }}" + version: "{{ cri_dockerd_version }}" + dest: "{{ local_release_dir }}/cri-dockerd-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz" + sha256: "{{ cri_dockerd_archive_checksum }}" + url: "{{ cri_dockerd_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - k8s_cluster + crun: file: true enabled: "{{ crun_enabled }}" diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 index c4f4cf044..c2030126b 100644 --- a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -12,7 +12,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --config={{ kube_config_dir }}/kubelet-config.yaml \ --kubeconfig={{ kube_config_dir }}/kubelet.conf \ {# end kubeadm specific settings #} -{% if container_manager == 'docker' %} +{% if container_manager == 'docker' and not cri_dockerd_enabled %} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ {% else %} --container-runtime=remote \ diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 2e672b419..8e08b65f3 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -263,6 +263,9 @@ deploy_container_engine: "{{ inventory_hostname in groups['k8s_cluster'] or etcd # Container for runtime container_manager: containerd +# Enable CRI Docker interface +cri_dockerd_enabled: false + # Enable Kata Containers as additional container runtime # When enabled, it requires `container_manager` different than Docker kata_containers_enabled: false @@ -288,6 +291,8 @@ cri_socket: >- /var/run/crio/crio.sock {%- elif container_manager == 'containerd' -%} /var/run/containerd/containerd.sock + {%- elif cri_dockerd_enabled -%} + /var/run/cri-dockerd.sock {%- else -%} /var/run/dockershim.sock {%- endif -%} diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index c0963815e..de7c9997a 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -4,7 +4,9 @@ name: "{{ item }}" state: stopped with_items: - - kubelet + - kubelet.service + - cri-dockerd.service + - cri-dockerd.socket failed_when: false tags: - services @@ -15,6 +17,8 @@ state: absent with_items: - kubelet.service + - cri-dockerd.service + - cri-dockerd.socket - calico-node.service - containerd.service.d/http-proxy.conf - crio.service.d/http-proxy.conf @@ -299,6 +303,7 @@ - /etc/pki/ca-trust/source/anchors/kube-ca.crt - /var/log/pods/ - "{{ bin_dir }}/kubelet" + - "{{ bin_dir }}/cri-dockerd" - "{{ bin_dir }}/etcd-scripts" - "{{ bin_dir }}/etcd" - "{{ bin_dir }}/etcd-events" diff --git a/tests/files/packet_almalinux8-docker.yml b/tests/files/packet_almalinux8-docker.yml index bcc69cd53..6fd1fc0d7 100644 --- a/tests/files/packet_almalinux8-docker.yml +++ b/tests/files/packet_almalinux8-docker.yml @@ -8,3 +8,4 @@ vm_memory: 3072Mi container_manager: docker etcd_deployment_type: docker resolvconf_mode: docker_dns +cri_dockerd_enabled: true diff --git a/tests/files/packet_ubuntu20-aio-docker.yml b/tests/files/packet_ubuntu20-aio-docker.yml index 4089a6605..df758d9b5 100644 --- a/tests/files/packet_ubuntu20-aio-docker.yml +++ b/tests/files/packet_ubuntu20-aio-docker.yml @@ -15,3 +15,4 @@ enable_nodelocaldns: False container_manager: docker etcd_deployment_type: docker resolvconf_mode: docker_dns +cri_dockerd_enabled: true