gVisor: initial support for gVisor container runtime (#7661)
* Docker/Containerd: move downloads urls to containerd-common * gVisor: initial support for gVisor container runtimepull/7727/head
Cristian Calin
GitHub
parent
3fe6dbb65c
commit
282a27a07c
|
|
@ -41,6 +41,7 @@
|
|||
* [Containerd](docs/containerd.md)
|
||||
* [CRI-O](docs/cri-o.md)
|
||||
* [Kata Containers](docs/kata-containers.md)
|
||||
* [gVisor](docs/gvisor.md)
|
||||
* Advanced
|
||||
* [Proxy](/docs/proxy.md)
|
||||
* [Downloads](docs/downloads.md)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
# gVisor
|
||||
|
||||
[gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system.
|
||||
|
||||
gVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers.
|
||||
|
||||
## Usage
|
||||
|
||||
To enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`.
|
||||
|
||||
Containerd support:
|
||||
|
||||
```yaml
|
||||
container_manager: containerd
|
||||
gvisor_enabled: true
|
||||
```
|
||||
|
|
@ -1,2 +1,17 @@
|
|||
---
|
||||
containerd_package: 'containerd.io'
|
||||
|
||||
# Fedora docker-ce repo
|
||||
docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
|
||||
docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
|
||||
# CentOS/RedHat docker-ce repo
|
||||
docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
|
||||
docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
|
||||
# Ubuntu docker-ce repo
|
||||
docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
|
||||
docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
|
||||
docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
|
||||
# Debian docker-ce repo
|
||||
docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
|
||||
docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
|
||||
docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
|
||||
|
|
|
|||
|
|
@ -35,6 +35,10 @@ oom_score = {{ containerd_oom_score }}
|
|||
{% if kata_containers_enabled %}
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
|
||||
runtime_type = "io.containerd.kata-qemu.v2"
|
||||
{% endif %}
|
||||
{% if gvisor_enabled %}
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
|
||||
runtime_type = "io.containerd.runsc.v1"
|
||||
{% endif %}
|
||||
[plugins."io.containerd.grpc.v1.cri".registry]
|
||||
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
|
||||
|
|
|
|||
|
|
@ -15,20 +15,6 @@ docker_cgroup_driver: systemd
|
|||
|
||||
yum_repo_dir: /etc/yum.repos.d
|
||||
|
||||
# Fedora docker-ce repo
|
||||
docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
|
||||
docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
|
||||
# CentOS/RedHat docker-ce repo
|
||||
docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
|
||||
docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
|
||||
# Ubuntu docker-ce repo
|
||||
docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
|
||||
docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
|
||||
docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
|
||||
# Debian docker-ce repo
|
||||
docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
|
||||
docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
|
||||
docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
|
||||
docker_bin_dir: "/usr/bin"
|
||||
|
||||
# flag to enable/disable docker cleanup
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
- name: Converge
|
||||
hosts: all
|
||||
become: true
|
||||
vars:
|
||||
gvisor_enabled: true
|
||||
container_manager: containerd
|
||||
roles:
|
||||
- role: kubespray-defaults
|
||||
- role: containerd
|
||||
- role: gvisor
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"cniVersion": "0.2.0",
|
||||
"name": "mynet",
|
||||
"type": "bridge",
|
||||
"bridge": "cni0",
|
||||
"isGateway": true,
|
||||
"ipMasq": true,
|
||||
"ipam": {
|
||||
"type": "host-local",
|
||||
"subnet": "172.19.0.0/24",
|
||||
"routes": [
|
||||
{
|
||||
"dst": "0.0.0.0/0"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"metadata": {
|
||||
"name": "gvisor1"
|
||||
},
|
||||
"image": {
|
||||
"image": "docker.io/library/hello-world:latest"
|
||||
},
|
||||
"log_path": "gvisor1.0.log",
|
||||
"linux": {}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"metadata": {
|
||||
"name": "gvisor1",
|
||||
"namespace": "default",
|
||||
"attempt": 1,
|
||||
"uid": "hdishd83djaidwnduwk28bcsb"
|
||||
},
|
||||
"linux": {},
|
||||
"log_directory": "/tmp"
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
---
|
||||
driver:
|
||||
name: vagrant
|
||||
provider:
|
||||
name: libvirt
|
||||
options:
|
||||
driver: kvm
|
||||
lint: |
|
||||
set -e
|
||||
yamllint -c ../../../.yamllint .
|
||||
platforms:
|
||||
- name: ubuntu20
|
||||
box: generic/ubuntu2004
|
||||
cpus: 1
|
||||
memory: 1024
|
||||
nested: true
|
||||
groups:
|
||||
- kube_control_plane
|
||||
- name: centos8
|
||||
box: generic/centos8
|
||||
cpus: 1
|
||||
memory: 1024
|
||||
nested: true
|
||||
groups:
|
||||
- kube_control_plane
|
||||
provisioner:
|
||||
name: ansible
|
||||
env:
|
||||
ANSIBLE_ROLES_PATH: ../../../../
|
||||
config_options:
|
||||
defaults:
|
||||
callback_whitelist: profile_tasks
|
||||
lint:
|
||||
name: ansible-lint
|
||||
options:
|
||||
c: ../../../.ansible-lint
|
||||
inventory:
|
||||
group_vars:
|
||||
all:
|
||||
become: true
|
||||
verifier:
|
||||
name: testinfra
|
||||
lint:
|
||||
name: flake8
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
---
|
||||
- name: Prepare generic
|
||||
hosts: all
|
||||
become: true
|
||||
roles:
|
||||
- role: kubespray-defaults
|
||||
- role: bootstrap-os
|
||||
- role: ../adduser
|
||||
user: "{{ addusers.kube }}"
|
||||
tasks:
|
||||
- include_tasks: "../../../../download/tasks/download_file.yml"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(downloads.cni) }}"
|
||||
|
||||
- name: Prepare container runtime
|
||||
hosts: all
|
||||
become: true
|
||||
vars:
|
||||
container_manager: containerd
|
||||
kube_network_plugin: cni
|
||||
roles:
|
||||
- role: kubespray-defaults
|
||||
- role: ../network_plugin/cni
|
||||
- role: crictl
|
||||
tasks:
|
||||
- name: Copy test container files
|
||||
copy:
|
||||
src: "{{ item }}"
|
||||
dest: "/tmp/{{ item }}"
|
||||
owner: root
|
||||
mode: 0644
|
||||
with_items:
|
||||
- container.json
|
||||
- sandbox.json
|
||||
- name: Create /etc/cni/net.d directory
|
||||
file:
|
||||
path: /etc/cni/net.d
|
||||
state: directory
|
||||
owner: kube
|
||||
mode: 0755
|
||||
- name: Setup CNI
|
||||
copy:
|
||||
src: "{{ item }}"
|
||||
dest: "/etc/cni/net.d/{{ item }}"
|
||||
owner: root
|
||||
mode: 0644
|
||||
with_items:
|
||||
- 10-mynet.conf
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
import os
|
||||
|
||||
import testinfra.utils.ansible_runner
|
||||
|
||||
testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
|
||||
os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
|
||||
|
||||
|
||||
def test_run(host):
|
||||
gvisorruntime = "/usr/local/bin/runsc"
|
||||
with host.sudo():
|
||||
cmd = host.command(gvisorruntime + " --version")
|
||||
assert cmd.rc == 0
|
||||
assert "runsc version" in cmd.stdout
|
||||
|
||||
|
||||
def test_run_pod(host):
|
||||
runtime = "runsc"
|
||||
|
||||
run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
|
||||
with host.sudo():
|
||||
cmd = host.command(run_command)
|
||||
assert cmd.rc == 0
|
||||
|
||||
with host.sudo():
|
||||
log_f = host.file("/tmp/gvisor1.0.log")
|
||||
|
||||
assert log_f.exists
|
||||
assert b"Hello from Docker!" in log_f.content
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
---
|
||||
- name: gVisor | Download runsc binary
|
||||
include_tasks: "../../../download/tasks/download_file.yml"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}"
|
||||
|
||||
- name: gVisor | Download containerd-shim-runsc-v1 binary
|
||||
include_tasks: "../../../download/tasks/download_file.yml"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}"
|
||||
|
||||
- name: gVisor | Copy binaries
|
||||
copy:
|
||||
src: "{{ local_release_dir }}/gvisor-{{ item }}"
|
||||
dest: "{{ bin_dir }}/{{ item }}"
|
||||
mode: 0755
|
||||
remote_src: yes
|
||||
with_items:
|
||||
- runsc
|
||||
- containerd-shim-runsc-v1
|
||||
|
|
@ -7,6 +7,14 @@ dependencies:
|
|||
- container-engine
|
||||
- kata-containers
|
||||
|
||||
- role: container-engine/gvisor
|
||||
when:
|
||||
- gvisor_enabled
|
||||
- container_manager in ['docker', 'containerd']
|
||||
tags:
|
||||
- container-engine
|
||||
- gvisor
|
||||
|
||||
- role: container-engine/crun
|
||||
when:
|
||||
- crun_enabled
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ kubeadm_version: "{{ kube_version }}"
|
|||
etcd_version: v3.4.13
|
||||
crun_version: 0.19
|
||||
kata_containers_version: 2.1.0
|
||||
gvisor_version: 20210518.0
|
||||
|
||||
# gcr and kubernetes image repo define
|
||||
gcr_image_repo: "gcr.io"
|
||||
|
|
@ -108,6 +109,8 @@ crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/down
|
|||
helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
|
||||
crun_download_url: "https://github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
|
||||
kata_containers_download_url: "https://github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
|
||||
# gVisor only supports amd64 and uses x86_64 to in the download link
|
||||
gvisor_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}"
|
||||
nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
|
||||
krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz"
|
||||
|
||||
|
|
@ -401,6 +404,22 @@ kata_containers_binary_checksums:
|
|||
2.0.4: 0
|
||||
2.1.0: 0
|
||||
|
||||
gvisor_runsc_binary_checksums:
|
||||
arm:
|
||||
20210518.0: 0
|
||||
arm64:
|
||||
20210518.0: 0
|
||||
amd64:
|
||||
20210518.0: 907811e10576d31b6f63d2f91f1c3be2b2df94e222fb48eb509fce48cd6ae9e0
|
||||
|
||||
gvisor_containerd_shim_binary_checksums:
|
||||
arm:
|
||||
20210518.0: 0
|
||||
arm64:
|
||||
20210518.0: 0
|
||||
amd64:
|
||||
20210518.0: d6deda79562cadf4fc98ccf1d6d9fd1d2dff3890b184c7b546167bd6eff241ec
|
||||
|
||||
nerdctl_archive_checksums:
|
||||
arm:
|
||||
0.8.1: 27bdad3f9e2667620f70617c48d595c5c4e24a10fbcd00d31202cd8d571c9233
|
||||
|
|
@ -420,6 +439,8 @@ crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
|
|||
helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}"
|
||||
crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}"
|
||||
kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}"
|
||||
gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}"
|
||||
gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}"
|
||||
nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}"
|
||||
krew_archive_checksum: "{{ krew_archive_checksums[krew_version] }}"
|
||||
|
||||
|
|
@ -715,6 +736,32 @@ downloads:
|
|||
groups:
|
||||
- k8s_cluster
|
||||
|
||||
gvisor_runsc:
|
||||
enabled: "{{ gvisor_enabled }}"
|
||||
file: true
|
||||
version: "{{ gvisor_version }}"
|
||||
dest: "{{ local_release_dir }}/gvisor-runsc"
|
||||
sha256: "{{ gvisor_runsc_binary_checksum }}"
|
||||
url: "{{ gvisor_download_url }}/runsc"
|
||||
unarchive: false
|
||||
owner: "root"
|
||||
mode: 755
|
||||
groups:
|
||||
- k8s_cluster
|
||||
|
||||
gvisor_containerd_shim:
|
||||
enabled: "{{ gvisor_enabled }}"
|
||||
file: true
|
||||
version: "{{ gvisor_version }}"
|
||||
dest: "{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1"
|
||||
sha256: "{{ gvisor_containerd_shim_binary_checksum }}"
|
||||
url: "{{ gvisor_download_url }}/containerd-shim-runsc-v1"
|
||||
unarchive: false
|
||||
owner: "root"
|
||||
mode: 755
|
||||
groups:
|
||||
- k8s_cluster
|
||||
|
||||
nerdctl:
|
||||
file: true
|
||||
enabled: "{{ nerdctl_enabled }}"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
- name: gVisor | Create addon dir
|
||||
file:
|
||||
path: "{{ kube_config_dir}}/addons/gvisor"
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0755
|
||||
recurse: true
|
||||
|
||||
- name: gVisor | Templates List
|
||||
set_fact:
|
||||
gvisor_templates:
|
||||
- { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass }
|
||||
|
||||
- name: gVisort | Create manifests
|
||||
template:
|
||||
src: "{{ item.file }}.j2"
|
||||
dest: "{{ kube_config_dir}}/addons/gvisor/{{ item.file }}"
|
||||
with_items: "{{ gvisor_templates }}"
|
||||
register: gvisor_manifests
|
||||
when:
|
||||
- inventory_hostname == groups['kube_control_plane'][0]
|
||||
|
||||
- name: gVisor | Apply manifests
|
||||
kube:
|
||||
name: "{{ item.item.name }}"
|
||||
kubectl: "{{ bin_dir}}/kubectl"
|
||||
resource: "{{ item.item.type }}"
|
||||
filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}"
|
||||
state: "latest"
|
||||
with_items: "{{ gvisor_manifests.results }}"
|
||||
when:
|
||||
- inventory_hostname == groups['kube_control_plane'][0]
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
kind: RuntimeClass
|
||||
apiVersion: node.k8s.io/v1{{ 'beta1' if kube_version is version('v1.20.0', '<') else '' }}
|
||||
metadata:
|
||||
name: gvisor
|
||||
handler: runsc
|
||||
|
|
@ -7,6 +7,13 @@ dependencies:
|
|||
- kata-containers
|
||||
- container-runtimes
|
||||
|
||||
- role: kubernetes-apps/container_runtimes/gvisor
|
||||
when: gvisor_enabled
|
||||
tags:
|
||||
- apps
|
||||
- gvisor
|
||||
- container-runtimes
|
||||
|
||||
- role: kubernetes-apps/container_runtimes/crun
|
||||
when: crun_enabled
|
||||
tags:
|
||||
|
|
|
|||
|
|
@ -312,6 +312,12 @@
|
|||
msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details"
|
||||
when: kata_containers_enabled
|
||||
|
||||
- name: Stop if gvisor_enabled is enabled when container_manager is not containerd
|
||||
assert:
|
||||
that: container_manager == 'containerd'
|
||||
msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details"
|
||||
when: gvisor_enabled
|
||||
|
||||
- name: Stop if download_localhost is enabled for Flatcar Container Linux
|
||||
assert:
|
||||
that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"]
|
||||
|
|
|
|||
|
|
@ -237,6 +237,10 @@ container_manager: docker
|
|||
# When enabled, it requires `container_manager` different than Docker
|
||||
kata_containers_enabled: false
|
||||
|
||||
# Enable gVisor as an additional container runtime
|
||||
# gVisor is only supported with container_manager Docker or containerd
|
||||
gvisor_enabled: false
|
||||
|
||||
# Enable crun as additional container runtime
|
||||
# When enabled, it requires container_manager=crio
|
||||
crun_enabled: false
|
||||
|
|
|
|||
Loading…
Reference in New Issue