Enable containerd to deploy vanilla containerd package (#4951)

* Enable containerd to deploy vanilla containerd package

Fixes kubeadm references to CRI socket for containerd
Fixes download role cache feature to work with containerd

Change-Id: I2ab8f0031107e2f0d1a85c39b4beb66f08509a01

* use containerd for flannel-addons job

Change-Id: Ied375c7d65e64a625ffbd995ff16f2374067dee6

* add containerd vars

Change-Id: Ib9a8a04e501c481a86235413cbec63f3672baf91

* fixup vars

Change-Id: Ibea64e4b18405a578b52a13da100384582aa24c2

* more fixes

* fix rh repo

Change-Id: I00575a77cfb7b81d6095db5d918a52023c8f13ba

* Adjust helm host install for containerd
pull/4964/head
Matthew Mosesohn 2019-07-11 09:46:54 +03:00 committed by Kubernetes Prow Robot
parent a038d62644
commit 29307740dd
24 changed files with 408 additions and 68 deletions

View File

@ -10,3 +10,35 @@ containerd_config:
registries:
"docker.io": "https://registry-1.docker.io"
max_container_log_line_size: -1
containerd_version: '1.2.6'
containerd_package: 'containerd.io'
containerd_cfg_dir: /etc/containerd
# Path to runc binray
runc_binary: /usr/sbin/runc
yum_repo_dir: /etc/yum.repos.d
yum_conf: /etc/yum.conf
containerd_yum_conf: /etc/yum_containerd.conf
# Optional values for containerd apt repo
containerd_package_info:
pkgs:
containerd_repo_key_info:
repo_keys:
containerd_repo_info:
repos:
extras_rh_repo_base_url: "http://mirror.centos.org/centos/$releasever/extras/$basearch/"
extras_rh_repo_gpgkey: "http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-7"
# Ubuntu docker-ce repo
containerd_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
containerd_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
containerd_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
containerd_ubuntu_repo_component: 'stable'

View File

@ -2,23 +2,19 @@
- name: restart containerd
command: /bin/true
notify:
- Containerd | reload containerd
- Containerd | pause while containerd restarts
- Containerd | restart containerd
- Containerd | wait for containerd
- name: Containerd | reload containerd
service:
- name: Containerd | restart containerd
systemd:
name: containerd
state: restarted
- name: Containerd | pause while containerd restarts
pause:
seconds: 5
prompt: "Waiting for containerd restart"
enabled: yes
daemon-reload: yes
- name: Containerd | wait for containerd
command: "{{ containerd_bin_dir }}/ctr images ls -q"
register: containerd_ready
retries: 10
delay: 5
retries: 8
delay: 4
until: containerd_ready.rc == 0

View File

@ -0,0 +1,106 @@
---
- name: ensure containerd repository public key is installed
action: "{{ containerd_repo_key_info.pkg_key }}"
args:
id: "{{ item }}"
url: "{{ containerd_repo_key_info.url }}"
state: present
register: keyserver_task_result
until: keyserver_task_result is succeeded
retries: 4
delay: "{{ retry_stagger | d(3) }}"
with_items: "{{ containerd_repo_key_info.repo_keys }}"
when:
- ansible_os_family in ['Ubuntu', 'Debian']
- not is_atomic
- name: ensure containerd repository is enabled
action: "{{ containerd_repo_info.pkg_repo }}"
args:
repo: "{{ item }}"
state: present
with_items: "{{ containerd_repo_info.repos }}"
when:
- ansible_os_family in ['Ubuntu', 'Debian']
- not is_atomic
- containerd_repo_info.repos|length > 0
# This is required to ensure any apt upgrade will not break kubernetes
- name: Set containerd pin priority to apt_preferences on Debian family
template:
src: "apt_preferences.d/debian_containerd.j2"
dest: "/etc/apt/preferences.d/containerd"
owner: "root"
mode: 0644
when:
- ansible_os_family in ['Ubuntu', 'Debian']
- not is_atomic
- name: ensure containerd repository public key is installed
action: "{{ containerd_repo_key_info.pkg_key }}"
args:
id: "{{ item }}"
url: "{{ containerd_repo_key_info.url }}"
state: present
register: keyserver_task_result
until: keyserver_task_result is succeeded
retries: 4
delay: "{{ retry_stagger | d(3) }}"
with_items: "{{ containerd_repo_key_info.repo_keys }}"
when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse", "ClearLinux"] or is_atomic)
- name: ensure containerd repository is enabled
action: "{{ containerd_repo_info.pkg_repo }}"
args:
repo: "{{ item }}"
state: present
with_items: "{{ containerd_repo_info.repos }}"
when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse", "ClearLinux"] or is_atomic) and (containerd_repo_info.repos|length > 0)
- name: Configure containerd repository on Fedora
template:
src: "fedora_containerd.repo.j2"
dest: "{{ yum_repo_dir }}/containerd.repo"
when: ansible_distribution == "Fedora" and not is_atomic
- name: Configure containerd repository on RedHat/CentOS
template:
src: "rh_containerd.repo.j2"
dest: "{{ yum_repo_dir }}/containerd.repo"
when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic
- name: check if container-selinux is available
yum:
list: "container-selinux"
register: yum_result
when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic
- name: Configure extras repository on RedHat/CentOS if container-selinux is not available in current repos
yum_repository:
name: extras
description: "CentOS-7 - Extras"
state: present
baseurl: "{{ extras_rh_repo_base_url }}"
file: "extras"
gpgcheck: yes
gpgkey: "{{ extras_rh_repo_gpgkey }}"
keepcache: "{{ containerd_rpm_keepcache | default('1') }}"
proxy: " {{ http_proxy | default('_none_') }}"
when:
- ansible_distribution in ["CentOS","RedHat"] and not is_atomic
- yum_result.results | length == 0
- name: Copy yum.conf for editing
copy:
src: "{{ yum_conf }}"
dest: "{{ containerd_yum_conf }}"
remote_src: yes
when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic
- name: Edit copy of yum.conf to set obsoletes=0
lineinfile:
path: "{{ containerd_yum_conf }}"
state: present
regexp: '^obsoletes='
line: 'obsoletes=0'
when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic

View File

@ -1,6 +1,6 @@
---
- name: crictl | Download crictl
include_tasks: "roles/download/tasks/download_file.yml"
include_tasks: "../../../download/tasks/download_file.yml"
vars:
download: "{{ download_defaults | combine(downloads.crictl) }}"

View File

@ -5,27 +5,119 @@
when:
- not ansible_distribution in ["CentOS","RedHat", "Ubuntu", "Debian"]
- name: Install Docker
include_role:
name: container-engine/docker
- name: gather os specific variables
include_vars: "{{ item }}"
with_first_found:
- files:
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}-{{ host_architecture }}.yml"
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}.yml"
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
- "{{ ansible_distribution|lower }}-{{ host_architecture }}.yml"
- "{{ ansible_distribution|lower }}.yml"
- "{{ ansible_os_family|lower }}-{{ host_architecture }}.yml"
- "{{ ansible_os_family|lower }}.yml"
- defaults.yml
paths:
- ../vars
skip: true
tags:
- facts
- name: Install config.toml
- include_tasks: containerd_repo.yml
- name: ensure containerd config directory
file:
dest: "{{ containerd_cfg_dir }}"
state: directory
mode: 0755
owner: root
group: root
- name: Copy containerd config file
template:
src: config.toml.j2
dest: /etc/containerd/config.toml
owner: bin
dest: "{{ containerd_cfg_dir }}/config.toml"
owner: "root"
mode: 0644
notify: restart containerd
- name: Stop and disabled Docker
systemd:
name: docker
state: stopped
enabled: no
- name: Restart containerd
systemd:
name: containerd
state: restarted
- name: ensure containerd repository public key is installed
action: "{{ containerd_repo_key_info.pkg_key }}"
args:
id: "{{ item }}"
url: "{{ containerd_repo_key_info.url }}"
state: present
register: keyserver_task_result
until: keyserver_task_result is succeeded
retries: 4
delay: "{{ retry_stagger | d(3) }}"
with_items: "{{ containerd_repo_key_info.repo_keys }}"
when:
- ansible_os_family in ['Ubuntu', 'Debian']
- not is_atomic
- name: ensure containerd repository is enabled
action: "{{ containerd_repo_info.pkg_repo }}"
args:
repo: "{{ item }}"
state: present
with_items: "{{ containerd_repo_info.repos }}"
when:
- ansible_os_family in ['Ubuntu', 'Debian']
- not is_atomic
- containerd_repo_info.repos|length > 0
# This is required to ensure any apt upgrade will not break kubernetes
- name: Set containerd pin priority to apt_preferences on Debian family
template:
src: "apt_preferences.d/debian_containerd.j2"
dest: "/etc/apt/preferences.d/containerd"
owner: "root"
mode: 0644
when:
- ansible_os_family in ['Ubuntu', 'Debian']
- not is_atomic
- name: ensure containerd packages are installed
action: "{{ containerd_package_info.pkg_mgr }}"
args:
pkg: "{{ item.name }}"
force: "{{ item.force | default(omit) }}"
conf_file: "{{ item.yum_conf | default(omit) }}"
state: present
update_cache: "{{ omit if ansible_distribution == 'Fedora' else True }}"
register: containerd_task_result
until: containerd_task_result is succeeded
retries: 4
delay: "{{ retry_stagger | d(3) }}"
with_items: "{{ containerd_package_info.pkgs }}"
notify: restart containerd
when:
- not is_atomic
- containerd_package_info.pkgs|length > 0
ignore_errors: true
- name: Check if runc is installed
stat:
path: /usr/sbin/runc
register: runc_stat
- name: Install runc package if necessary
action: "{{ containerd_package_info.pkg_mgr }}"
args:
pkg: runc
state: present
update_cache: "{{ omit if ansible_distribution == 'Fedora' else True }}"
register: runc_task_result
until: runc_task_result is succeeded
retries: 4
delay: "{{ retry_stagger | d(3) }}"
notify: restart containerd
when:
- not is_atomic
- not runc_stat.stat.exists
- name: Install crictl config
template:
@ -35,16 +127,6 @@
mode: 0644
- name: Install crictl completion
shell: /usr/local/bin/crictl completion >/etc/bash_completion.d/crictl
shell: "{{ bin_dir }}/crictl completion >/etc/bash_completion.d/crictl"
ignore_errors: True
when: ansible_distribution in ["CentOS","RedHat", "Ubuntu", "Debian"]
- name: Enable containerd
systemd:
name: containerd.service
state: started
enabled: yes
daemon-reload: yes
- name: flush handlers so we can wait for containerd to come up
meta: flush_handlers

View File

@ -0,0 +1,3 @@
Package: {{ containerd_package }}
Pin: version {{ containerd_version }}.*
Pin-Priority: 1001

View File

@ -13,7 +13,7 @@ disabled_plugins = ["restart"]
[plugins.linux]
shim = "/usr/bin/containerd-shim"
runtime = "/usr/sbin/runc"
runtime = "{{ runc_binary }}"
[plugins.cri]
stream_server_address = "127.0.0.1"

View File

@ -0,0 +1,17 @@
[docker-ce]
name=Docker-CE Repository
baseurl={{ docker_rh_repo_base_url }}
enabled=1
gpgcheck=1
keepcache={{ docker_rpm_keepcache | default('1') }}
gpgkey={{ docker_rh_repo_gpgkey }}
{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}
[docker-engine]
name=Docker-Engine Repository
baseurl={{ dockerproject_rh_repo_base_url }}
enabled=1
gpgcheck=1
keepcache={{ docker_rpm_keepcache | default('1') }}
gpgkey={{ dockerproject_rh_repo_gpgkey }}
{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}

View File

@ -0,0 +1,28 @@
---
containerd_versioned_pkg:
'latest': "{{ containerd_package }}"
'1.2.4': "{{ containerd_package }}-1.2.4-3.1.el7"
'1.2.5': "{{ containerd_package }}-1.2.5-3.1.el7"
'1.2.6': "{{ containerd_package }}-1.2.6-3.3.el7"
'stable': "{{ containerd_package }}-1.2.6-3.3.el7"
'edge': "{{ containerd_package }}-1.2.6-3.3.el7"
containerd_package_info:
pkg_mgr: yum
pkgs:
- name: "{{ containerd_versioned_pkg[containerd_version | string] }}"
containerd_pkgs:
- name: "{{ containerd_versioned_pkg[containerd_version | string] }}"
yum_conf: "{{ containerd_yum_conf }}"
containerd_repo_key_info:
pkg_key: ''
repo_keys: []
containerd_repo_info:
pkg_repo: ''
repos: []
runc_binary: /bin/runc

View File

@ -0,0 +1,17 @@
---
# docker-ce containerd.io does not contain daemon
containerd_package: containerd
containerd_package_info:
pkg_mgr: zypper
pkgs:
- name: "{{ containerd_package }}"
state: latest
containerd_repo_key_info:
pkg_key: ''
repo_keys: []
containerd_repo_info:
pkg_repo: ''
repos: []

View File

@ -0,0 +1,27 @@
---
containerd_versioned_pkg:
'latest': "{{ containerd_package }}"
'1.2.4': "{{ containerd_package }}=1.2.4-1"
'stable': "{{ containerd_package }}=1.2.4-1"
'edge': "{{ containerd_package }}=1.2.4-1"
containerd_package_info:
pkg_mgr: apt
pkgs:
- name: "{{ containerd_versioned_pkg[containerd_version | string] }}"
force: true
containerd_repo_key_info:
pkg_key: apt_key
url: '{{ containerd_ubuntu_repo_gpgkey }}'
repo_keys:
- '{{ containerd_ubuntu_repo_repokey }}'
containerd_repo_info:
pkg_repo: apt_repository
repos:
- >
deb {{ containerd_ubuntu_repo_base_url }}
{{ ansible_distribution_release|lower }}
{{ containerd_ubuntu_repo_component }}

View File

@ -20,10 +20,3 @@ dependencies:
tags:
- container-engine
- docker
- role: container-engine/containerd
when:
- container_manager == 'containerd'
tags:
- container-engine
- containerd

View File

@ -5,8 +5,7 @@
# It will output something like the following:
# nginx:1.15,gcr.io/google-containers/kube-proxy:v1.14.1,gcr.io/google-containers/kube-proxy@sha256:44af2833c6cbd9a7fc2e9d2f5244a39dfd2e31ad91bf9d4b7d810678db738ee9,gcr.io/google-containers/kube-apiserver:v1.14.1,etc...
- name: check_pull_required | Generate a list of information about the images on a node
shell: >-
{{ docker_bin_dir }}/docker images -q | xargs -r {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} if .RepoTags {{ '}}' }}{{ '{{' }} (index .RepoTags) {{ '}}' }}{{ '{{' }} end {{ '}}' }}{{ '{{' }} if .RepoDigests {{ '}}' }},{{ '{{' }} (index .RepoDigests) {{ '}}' }}{{ '{{' }} end {{ '}}' }}" | sed -e 's/^ *\[//g' -e 's/\] *$//g' -e 's/ /\n/g' | tr '\n' ','
shell: "{{ image_info_command }}"
delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
no_log: true
register: docker_images

View File

@ -65,7 +65,7 @@
- ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
- name: download_container | Prepare container download
import_tasks: check_pull_required.yml
include_tasks: check_pull_required.yml
run_once: "{{ download_run_once }}"
when:
- not download_always_pull

View File

@ -24,10 +24,11 @@
- include_tasks: ../../container-engine/containerd/tasks/crictl.yml
when:
- not skip_downloads|default(false)
- container_manager in ['containerd', 'crio']
- name: download | Get kubeadm binary and list of required images
import_tasks: prep_kubeadm_images.yml
include_tasks: prep_kubeadm_images.yml
when:
- kube_version is version('v1.11.0', '>=')
- not skip_downloads|default(false)

View File

@ -5,6 +5,20 @@
tags:
- facts
- name: Set image info command for containerd
set_fact:
image_info_command: "{{ containerd_bin_dir }}/ctr images ls | tail -n +2 | awk -F '[ :]+' '{print $1\":\"$2\",\"$1\":\"$4\"@\"$5}' | tr '\n' ','"
when: container_manager == 'containerd'
- name: Register docker images info
shell: "{{ image_info_command }}"
no_log: true
register: docker_images
failed_when: false
changed_when: false
check_mode: no
when: download_container
- name: prep_download | Create staging directory on remote node
file:
path: "{{ local_release_dir }}/images"

View File

@ -13,3 +13,9 @@ etcd:
{% for endpoint in etcd_access_addresses.split(',') %}
- {{ endpoint }}
{% endfor %}
{% if dns_mode in ['coredns', 'coredns_dual'] %}
dns:
type: CoreDNS
imageRepository: {{ coredns_image_repo | regex_replace('/coredns$','') }}
imageTag: {{ coredns_image_tag }}
{% endif %}

View File

@ -1,6 +1,25 @@
---
- name: Helm | Set commands for helm host tasks
set_fact:
helm_compare_command: >-
{%- if container_manager in ['docker', 'crio'] %}
{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir --entrypoint /usr/bin/cmp {{ helm_image_repo }}:{{ helm_image_tag }} /usr/local/bin/helm /systembindir/helm
{%- elif container_manager == "containerd" %}
ctr run --rm --mount type=bind,src={{ bin_dir }},dst=/systembindir,options=rbind:rw {{ helm_image_repo }}:{{ helm_image_tag }} helm-compare sh -c 'cmp /usr/local/bin/helm /systembindir/helm'
{%- endif %}
helm_copy_command: >-
{%- if container_manager in ['docker', 'crio'] %}
{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir --entrypoint /bin/cp {{ helm_image_repo }}:{{ helm_image_tag }} -f /usr/local/bin/helm /systembindir/helm
{%- elif container_manager == "containerd" %}
ctr run --rm --mount type=bind,src={{ bin_dir }},dst=/systembindir,options=rbind:rw {{ helm_image_repo }}:{{ helm_image_tag }} helm-copy sh -c '/bin/cp -f /usr/local/bin/helm /systembindir/helm'
{%- endif %}
- name: Helm | ensure helm container is pulled for containerd
command: "ctr i pull {{ helm_image_repo }}:{{ helm_image_tag }}"
when: container_manager == "containerd"
- name: Helm | Compare host helm with hyperkube container
command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir --entrypoint /usr/bin/cmp {{ helm_image_repo }}:{{ helm_image_tag }} /usr/local/bin/helm /systembindir/helm"
command: "{{ helm_compare_command }}"
register: helm_task_compare_result
until: helm_task_compare_result.rc in [0,1,2]
retries: 4
@ -9,7 +28,7 @@
failed_when: "helm_task_compare_result.rc not in [0,1,2]"
- name: Helm | Copy helm from helm container
command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir --entrypoint /bin/cp {{ helm_image_repo }}:{{ helm_image_tag }} -f /usr/local/bin/helm /systembindir/helm"
command: "{{ helm_copy_command }}"
when: helm_task_compare_result.rc != 0
register: helm_task_result
until: helm_task_result.rc == 0

View File

@ -40,6 +40,12 @@
run_once: yes
when: kubeconfig_localhost|default(false)
- name: Wait for k8s apiserver
wait_for:
host: "{{ kube_apiserver_access_address }}"
port: "{{ kube_apiserver_port }}"
timeout: 180
# NOTE(mattymo): Please forgive this workaround
- name: Generate admin kubeconfig with external api endpoint
shell: >-

View File

@ -17,11 +17,7 @@ nodeRegistration:
{% else %}
taints: []
{% endif %}
{% if container_manager == 'crio' %}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}
criSocket: {{ cri_socket }}
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration

View File

@ -5,7 +5,7 @@ discovery:
{% if kubeadm_config_api_fqdn is defined %}
apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
{% else %}
apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}}
apiServerEndpoint: {{ kubeadm_discovery_address }}
{% endif %}
token: {{ kubeadm_token }}
unsafeSkipCAVerification: true
@ -18,8 +18,4 @@ controlPlane:
certificateKey: {{ kubeadm_certificate_key }}
nodeRegistration:
name: {{ kube_override_hostname|default(inventory_hostname) }}
{% if container_manager == 'crio' %}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}
criSocket: {{ cri_socket }}

View File

@ -5,25 +5,25 @@ cloud_region: us-central1-c
cloud_machine_type: "n1-standard-2"
mode: ha
# Deployment settings
# Kubespray settings
kubeadm_control_plane: true
kubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c06d741085
kube_proxy_mode: iptables
kube_network_plugin: flannel
helm_enabled: true
kubernetes_audit: true
container_manager: containerd
etcd_events_cluster_enabled: true
local_volume_provisioner_enabled: true
etcd_deployment_type: host
deploy_netchecker: true
dns_min_replicas: 1
cloud_provider: gce
kube_encrypt_secret_data: true
# ingress_nginx_enabled: true
ingress_nginx_enabled: true
cert_manager_enabled: true
# Disabled temporarily
# Disable as health checks are still unstable and slow to respond.
metrics_server_enabled: false
metrics_server_kubelet_insecure_tls: true
kube_token_auth: true
kube_basic_auth: true
enable_nodelocaldns: false
local_path_provisioner_enabled: true

View File

@ -10,6 +10,7 @@ kube_proxy_mode: iptables
kube_network_plugin: flannel
helm_enabled: true
kubernetes_audit: true
container_manager: containerd
etcd_events_cluster_enabled: true
local_volume_provisioner_enabled: true
etcd_deployment_type: host

View File

@ -5,5 +5,6 @@ mode: default
# Kubespray settings
kube_network_plugin: canal
container_manager: containerd
deploy_netchecker: true
dns_min_replicas: 1