gVisor: initial support for gVisor container runtime (#7661)

* Docker/Containerd: move downloads urls to containerd-common

* gVisor: initial support for gVisor container runtime

docs/_sidebar.md

@@ -41,6 +41,7 @@
   * [Containerd](docs/containerd.md)
   * [CRI-O](docs/cri-o.md)
   * [Kata Containers](docs/kata-containers.md)
+  * [gVisor](docs/gvisor.md)
 * Advanced
   * [Proxy](/docs/proxy.md)
   * [Downloads](docs/downloads.md)

docs/gvisor.md

@@ -0,0 +1,16 @@
+# gVisor
+
+[gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system.
+
+gVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers.
+
+## Usage
+
+To enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`.
+
+Containerd support:
+
+```yaml
+container_manager: containerd
+gvisor_enabled: true
+```

roles/container-engine/containerd-common/defaults/main.yml

@@ -1,2 +1,17 @@
 ---
 containerd_package: 'containerd.io'
+
+# Fedora docker-ce repo
+docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
+docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
+# CentOS/RedHat docker-ce repo
+docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
+docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
+# Ubuntu docker-ce repo
+docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
+docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
+docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
+# Debian docker-ce repo
+docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
+docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
+docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'

roles/container-engine/containerd/templates/config.toml.j2

@@ -35,6 +35,10 @@ oom_score = {{ containerd_oom_score }}
 {% if kata_containers_enabled %}
         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
           runtime_type = "io.containerd.kata-qemu.v2"
+{% endif %}
+{% if gvisor_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
+          runtime_type = "io.containerd.runsc.v1"
 {% endif %}
     [plugins."io.containerd.grpc.v1.cri".registry]
       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]

roles/container-engine/docker/defaults/main.yml

@@ -15,20 +15,6 @@ docker_cgroup_driver: systemd
 
 yum_repo_dir: /etc/yum.repos.d
 
-# Fedora docker-ce repo
-docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
-docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
-# CentOS/RedHat docker-ce repo
-docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
-docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
-# Ubuntu docker-ce repo
-docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
-docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
-docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
-# Debian docker-ce repo
-docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
-docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
-docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
 docker_bin_dir: "/usr/bin"
 
 # flag to enable/disable docker cleanup

roles/container-engine/gvisor/molecule/default/converge.yml

@@ -0,0 +1,11 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    gvisor_enabled: true
+    container_manager: containerd
+  roles:
+    - role: kubespray-defaults
+    - role: containerd
+    - role: gvisor

roles/container-engine/gvisor/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

roles/container-engine/gvisor/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "gvisor1"
+  },
+  "image": {
+    "image": "docker.io/library/hello-world:latest"
+  },
+  "log_path": "gvisor1.0.log",
+  "linux": {}
+}

roles/container-engine/gvisor/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "gvisor1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

roles/container-engine/gvisor/molecule/default/molecule.yml

@@ -0,0 +1,44 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+    options:
+      driver: kvm
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+  - name: centos8
+    box: generic/centos8
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callback_whitelist: profile_tasks
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+  inventory:
+    group_vars:
+      all:
+        become: true
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

roles/container-engine/gvisor/molecule/default/prepare.yml

@@ -0,0 +1,48 @@
+---
+- name: Prepare generic
+  hosts: all
+  become: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: ../adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare container runtime
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: ../network_plugin/cni
+    - role: crictl
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: kube
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

roles/container-engine/gvisor/molecule/default/tests/test_default.py

@@ -0,0 +1,29 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_run(host):
+    gvisorruntime = "/usr/local/bin/runsc"
+    with host.sudo():
+        cmd = host.command(gvisorruntime + " --version")
+    assert cmd.rc == 0
+    assert "runsc version" in cmd.stdout
+
+
+def test_run_pod(host):
+    runtime = "runsc"
+
+    run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/gvisor1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker!" in log_f.content

roles/container-engine/gvisor/tasks/main.yml

@@ -0,0 +1,20 @@
+---
+- name: gVisor | Download runsc binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}"
+
+- name: gVisor | Download containerd-shim-runsc-v1 binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}"
+
+- name: gVisor | Copy binaries
+  copy:
+    src: "{{ local_release_dir }}/gvisor-{{ item }}"
+    dest: "{{ bin_dir }}/{{ item }}"
+    mode: 0755
+    remote_src: yes
+  with_items:
+    - runsc
+    - containerd-shim-runsc-v1

roles/container-engine/meta/main.yml

@@ -7,6 +7,14 @@ dependencies:
       - container-engine
       - kata-containers
 
+  - role: container-engine/gvisor
+    when:
+      - gvisor_enabled
+      - container_manager in ['docker', 'containerd']
+    tags:
+      - container-engine
+      - gvisor
+
   - role: container-engine/crun
     when:
       - crun_enabled

roles/download/defaults/main.yml

@@ -53,6 +53,7 @@ kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.4.13
 crun_version: 0.19
 kata_containers_version: 2.1.0
+gvisor_version: 20210518.0
 
 # gcr and kubernetes image repo define
 gcr_image_repo: "gcr.io"
@@ -108,6 +109,8 @@ crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/down
 helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
 crun_download_url: "https://github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
 kata_containers_download_url: "https://github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
+# gVisor only supports amd64 and uses x86_64 to in the download link
+gvisor_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}"
 nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz"
 
@@ -401,6 +404,22 @@ kata_containers_binary_checksums:
     2.0.4: 0
     2.1.0: 0
 
+gvisor_runsc_binary_checksums:
+  arm:
+    20210518.0: 0
+  arm64:
+    20210518.0: 0
+  amd64:
+    20210518.0: 907811e10576d31b6f63d2f91f1c3be2b2df94e222fb48eb509fce48cd6ae9e0
+
+gvisor_containerd_shim_binary_checksums:
+  arm:
+    20210518.0: 0
+  arm64:
+    20210518.0: 0
+  amd64:
+    20210518.0: d6deda79562cadf4fc98ccf1d6d9fd1d2dff3890b184c7b546167bd6eff241ec
+
 nerdctl_archive_checksums:
   arm:
     0.8.1: 27bdad3f9e2667620f70617c48d595c5c4e24a10fbcd00d31202cd8d571c9233
@@ -420,6 +439,8 @@ crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
 helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}"
 crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}"
 kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}"
+gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}"
+gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}"
 nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}"
 krew_archive_checksum: "{{ krew_archive_checksums[krew_version] }}"
 
@@ -715,6 +736,32 @@ downloads:
     groups:
     - k8s_cluster
 
+  gvisor_runsc:
+    enabled: "{{ gvisor_enabled }}"
+    file: true
+    version: "{{ gvisor_version }}"
+    dest: "{{ local_release_dir }}/gvisor-runsc"
+    sha256: "{{ gvisor_runsc_binary_checksum }}"
+    url: "{{ gvisor_download_url }}/runsc"
+    unarchive: false
+    owner: "root"
+    mode: 755
+    groups:
+    - k8s_cluster
+
+  gvisor_containerd_shim:
+    enabled: "{{ gvisor_enabled }}"
+    file: true
+    version: "{{ gvisor_version }}"
+    dest: "{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1"
+    sha256: "{{ gvisor_containerd_shim_binary_checksum }}"
+    url: "{{ gvisor_download_url }}/containerd-shim-runsc-v1"
+    unarchive: false
+    owner: "root"
+    mode: 755
+    groups:
+    - k8s_cluster
+
   nerdctl:
     file: true
     enabled: "{{ nerdctl_enabled }}"

roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml

@@ -0,0 +1,33 @@
+---
+- name: gVisor | Create addon dir
+  file:
+    path: "{{ kube_config_dir}}/addons/gvisor"
+    owner: root
+    group: root
+    mode: 0755
+    recurse: true
+
+- name: gVisor | Templates List
+  set_fact:
+    gvisor_templates:
+      - { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass }
+
+- name: gVisort | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir}}/addons/gvisor/{{ item.file }}"
+  with_items: "{{ gvisor_templates }}"
+  register: gvisor_manifests
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: gVisor | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    kubectl: "{{ bin_dir}}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ gvisor_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2

@@ -0,0 +1,6 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1{{ 'beta1' if kube_version is version('v1.20.0', '<') else '' }}
+metadata:
+  name: gvisor
+handler: runsc

roles/kubernetes-apps/container_runtimes/meta/main.yml

@@ -7,6 +7,13 @@ dependencies:
       - kata-containers
       - container-runtimes
 
+  - role: kubernetes-apps/container_runtimes/gvisor
+    when: gvisor_enabled
+    tags:
+      - apps
+      - gvisor
+      - container-runtimes
+
   - role: kubernetes-apps/container_runtimes/crun
     when: crun_enabled
     tags:

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -312,6 +312,12 @@
     msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details"
   when: kata_containers_enabled
 
+- name: Stop if gvisor_enabled is enabled when container_manager is not containerd
+  assert:
+    that: container_manager == 'containerd'
+    msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details"
+  when: gvisor_enabled
+
 - name: Stop if download_localhost is enabled for Flatcar Container Linux
   assert:
     that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"]

roles/kubespray-defaults/defaults/main.yaml

@@ -237,6 +237,10 @@ container_manager: docker
 # When enabled, it requires `container_manager` different than Docker
 kata_containers_enabled: false
 
+# Enable gVisor as an additional container runtime
+# gVisor is only supported with container_manager Docker or containerd
+gvisor_enabled: false
+
 # Enable crun as additional container runtime
 # When enabled, it requires container_manager=crio
 crun_enabled: false