From 6090af29e7ea23a9614e1da8376a5dc30d91f404 Mon Sep 17 00:00:00 2001 From: Takashi Okamoto Date: Thu, 9 Aug 2018 13:53:11 -0400 Subject: [PATCH] Add cri-o role. --- cluster.yml | 1 + roles/cri-o/defaults/main.yml | 2 + roles/cri-o/tasks/main.yaml | 40 +++ roles/cri-o/templates/crio.conf.j2 | 234 ++++++++++++++++++ roles/kubernetes/node/defaults/main.yml | 5 + .../node/templates/kubelet.kubeadm.env.j2 | 1 + scale.yml | 1 + upgrade-cluster.yml | 1 + 8 files changed, 285 insertions(+) create mode 100644 roles/cri-o/defaults/main.yml create mode 100644 roles/cri-o/tasks/main.yaml create mode 100644 roles/cri-o/templates/crio.conf.j2 diff --git a/cluster.yml b/cluster.yml index 8462ea894..14a4a6d37 100644 --- a/cluster.yml +++ b/cluster.yml @@ -34,6 +34,7 @@ - { role: kubespray-defaults} - { role: kubernetes/preinstall, tags: preinstall } - { role: docker, tags: docker, when: manage_docker|default(true) } + - { role: cri-o, tags: crio, when: manage_crio } - role: rkt tags: rkt when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]" diff --git a/roles/cri-o/defaults/main.yml b/roles/cri-o/defaults/main.yml new file mode 100644 index 000000000..3ae39da22 --- /dev/null +++ b/roles/cri-o/defaults/main.yml @@ -0,0 +1,2 @@ +--- +crio_rhel_repo_base_url: 'https://cbs.centos.org/repos/paas7-openshift-origin311-candidate/x86_64/os/' diff --git a/roles/cri-o/tasks/main.yaml b/roles/cri-o/tasks/main.yaml new file mode 100644 index 000000000..7edfd035e --- /dev/null +++ b/roles/cri-o/tasks/main.yaml @@ -0,0 +1,40 @@ +--- +- name: Add OpenShift Origin repository + yum_repository: + name: origin + description: OpenShift Origin Repo + baseurl: "{{ crio_rhel_repo_base_url }}" + gpgcheck: no + when: + - ansible_os_family == 'RedHat' + +- name: Install cri-o + package: + name: "{{ item }}" + state: present + with_items: + - cri-o + - cri-tools + - oci-systemd-hook + +- name: Install cri-o config + template: + src: crio.conf.j2 + dest: /etc/crio/crio.conf + +- name: Copy mounts.conf + shell: | + cp -T /usr/share/containers/mounts.conf /etc/containers/mounts.conf + +- name: Create directory for oci hooks + file: + path: /etc/containers/oci/hooks.d + state: directory + owner: root + mode: 0755 + +- name: Install cri-o service + service: + name: crio + enabled: yes + state: restarted diff --git a/roles/cri-o/templates/crio.conf.j2 b/roles/cri-o/templates/crio.conf.j2 new file mode 100644 index 000000000..b20a50c70 --- /dev/null +++ b/roles/cri-o/templates/crio.conf.j2 @@ -0,0 +1,234 @@ + +# The "crio" table contains all of the server options. +[crio] + +# CRI-O reads its storage defaults from the containers/storage configuration +# file, /etc/containers/storage.conf. Modify storage.conf if you want to +# change default storage for all tools that use containers/storage. If you +# want to modify just crio, you can change the storage configuration in this +# file. + +# root is a path to the "root directory". CRIO stores all of its data, +# including container images, in this directory. +#root = "/var/lib/containers/storage" + +# run is a path to the "run directory". CRIO stores all of its state +# in this directory. +#runroot = "/var/run/containers/storage" + +# storage_driver select which storage driver is used to manage storage +# of images and containers. +storage_driver = "overlay2" + +# storage_option is used to pass an option to the storage driver. +#storage_option = [ +#] + +# The "crio.api" table contains settings for the kubelet/gRPC interface. +[crio.api] + +# listen is the path to the AF_LOCAL socket on which crio will listen. +listen = "/var/run/crio/crio.sock" + +# stream_address is the IP address on which the stream server will listen +stream_address = "" + +# stream_port is the port on which the stream server will listen +stream_port = "10010" + +# stream_enable_tls enables encrypted tls transport of the stream server +stream_enable_tls = false + +# stream_tls_cert is the x509 certificate file path used to serve the encrypted stream. +# This file can change, and CRIO will automatically pick up the changes within 5 minutes. +stream_tls_cert = "" + +# stream_tls_key is the key file path used to serve the encrypted stream. +# This file can change, and CRIO will automatically pick up the changes within 5 minutes. +stream_tls_key = "" + +# stream_tls_ca is the x509 CA(s) file used to verify and authenticate client +# communication with the tls encrypted stream. +# This file can change, and CRIO will automatically pick up the changes within 5 minutes. +stream_tls_ca = "" + +# file_locking is whether file-based locking will be used instead of +# in-memory locking +file_locking = true + +# The "crio.runtime" table contains settings pertaining to the OCI +# runtime used and options for how to set up and manage the OCI runtime. +[crio.runtime] + +# runtime is the OCI compatible runtime used for trusted container workloads. +# This is a mandatory setting as this runtime will be the default one +# and will also be used for untrusted container workloads if +# runtime_untrusted_workload is not set. +runtime = "/usr/bin/runc" + +# runtime_untrusted_workload is the OCI compatible runtime used for untrusted +# container workloads. This is an optional setting, except if +# default_container_trust is set to "untrusted". +runtime_untrusted_workload = "" + +# default_workload_trust is the default level of trust crio puts in container +# workloads. It can either be "trusted" or "untrusted", and the default +# is "trusted". +# Containers can be run through different container runtimes, depending on +# the trust hints we receive from kubelet: +# - If kubelet tags a container workload as untrusted, crio will try first to +# run it through the untrusted container workload runtime. If it is not set, +# crio will use the trusted runtime. +# - If kubelet does not provide any information about the container workload trust +# level, the selected runtime will depend on the default_container_trust setting. +# If it is set to "untrusted", then all containers except for the host privileged +# ones, will be run by the runtime_untrusted_workload runtime. Host privileged +# containers are by definition trusted and will always use the trusted container +# runtime. If default_container_trust is set to "trusted", crio will use the trusted +# container runtime for all containers. +default_workload_trust = "trusted" + +# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE +no_pivot = false + +# conmon is the path to conmon binary, used for managing the runtime. +conmon = "/usr/libexec/crio/conmon" + +# conmon_env is the environment variable list for conmon process, +# used for passing necessary environment variable to conmon or runtime. +conmon_env = [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] + +# selinux indicates whether or not SELinux will be used for pod +# separation on the host. If you enable this flag, SELinux must be running +# on the host. +selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }} + +# seccomp_profile is the seccomp json profile path which is used as the +# default for the runtime. +seccomp_profile = "/etc/crio/seccomp.json" + +# apparmor_profile is the apparmor profile name which is used as the +# default for the runtime. +apparmor_profile = "crio-default" + +# cgroup_manager is the cgroup management implementation to be used +# for the runtime. +cgroup_manager = "cgroupfs" + +# default_capabilities is the list of capabilities to add and can be modified here. +# If capabilities below is commented out, the default list of capabilities defined in the +# spec will be added. +# If capabilities is empty below, only the capabilities defined in the container json +# file by the user/kube will be added. +default_capabilities = [ + "CHOWN", + "DAC_OVERRIDE", + "FSETID", + "FOWNER", + "NET_RAW", + "SETGID", + "SETUID", + "SETPCAP", + "NET_BIND_SERVICE", + "SYS_CHROOT", + "KILL", +] + +# hooks_dir_path is the oci hooks directory for automatically executed hooks +hooks_dir_path = "/usr/share/containers/oci/hooks.d" + +# default_mounts is the mounts list to be mounted for the container when created +# deprecated, will be taken out in future versions, add default mounts to either +# /usr/share/containers/mounts.conf or /etc/containers/mounts.conf +default_mounts = [ +] + +# CRI-O reads its default mounts from the following two files: +# 1) /etc/containers/mounts.conf - this is the override file, where users can +# either add in their own default mounts, or override the default mounts shipped +# with the package. +# 2) /usr/share/containers/mounts.conf - this is the default file read for mounts. +# If you want CRI-O to read from a different, specific mounts file, you can change +# the default_mounts_file path right below. Note, if this is done, CRI-O will only add +# mounts it finds in this file. + +# default_mounts_file is the file path holding the default mounts to be mounted for the +# container when created. +# default_mounts_file = "" + +# pids_limit is the number of processes allowed in a container +pids_limit = 1024 + +# log_size_max is the max limit for the container log size in bytes. +# Negative values indicate that no limit is imposed. +log_size_max = -1 + +# read-only indicates whether all containers will run in read-only mode +read_only = false + +# The "crio.image" table contains settings pertaining to the +# management of OCI images. + +# uid_mappings specifies the UID mappings to have in the user namespace. +# A range is specified in the form containerUID:HostUID:Size. Multiple +# ranges are separed by comma. +uid_mappings = "" + +# gid_mappings specifies the GID mappings to have in the user namespace. +# A range is specified in the form containerGID:HostGID:Size. Multiple +# ranges are separed by comma. +gid_mappings = "" + +[crio.image] + +# default_transport is the prefix we try prepending to an image name if the +# image name as we receive it can't be parsed as a valid source reference +default_transport = "docker://" + +# pause_image is the image which we use to instantiate infra containers. +pause_image = "docker://k8s.gcr.io/pause:3.1" + +# pause_command is the command to run in a pause_image to have a container just +# sit there. If the image contains the necessary information, this value need +# not be specified. +pause_command = "/pause" + +# signature_policy is the name of the file which decides what sort of policy we +# use when deciding whether or not to trust an image that we've pulled. +# Outside of testing situations, it is strongly advised that this be left +# unspecified so that the default system-wide policy will be used. +signature_policy = "" + +# image_volumes controls how image volumes are handled. +# The valid values are mkdir and ignore. +image_volumes = "mkdir" + +# CRI-O reads its configured registries defaults from the containers/image configuration +# file, /etc/containers/registries.conf. Modify registries.conf if you want to +# change default registries for all tools that use containers/image. If you +# want to modify just crio, you can change the registies configuration in this +# file. + +# insecure_registries is used to skip TLS verification when pulling images. +insecure_registries = [ + "{{ kube_service_addresses }}" +] + +# registries is used to specify a comma separated list of registries to be used +# when pulling an unqualified image (e.g. fedora:rawhide). +registries = [ + "docker.io" +] + +# The "crio.network" table contains settings pertaining to the +# management of CNI plugins. +[crio.network] + +# network_dir is is where CNI network configuration +# files are stored. +network_dir = "/etc/cni/net.d/" + +# plugin_dir is is where CNI plugin binaries are stored. +plugin_dir = "/opt/cni/bin/" diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 2c541c112..0e73d7932 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -31,6 +31,11 @@ kubelet_cgroups_per_qos: true # Set to empty to avoid cgroup creation kubelet_enforce_node_allocatable: "\"\"" +# Set runtime cgroups +kubelet_runtime_cgroups: "/systemd/system.slice" +# Set kubelet cgroups +kubelet_kubelet_cgroups: "/systemd/system.slice" + # Set false to enable sharing a pid namespace between containers in a pod. # Note that PID namespace sharing requires docker >= 1.13.1. kubelet_disable_shared_pid: true diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index aca97ae12..46b5e40d7 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -42,6 +42,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% else %} --fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \ {% endif %} +--runtime-cgroups={{ kubelet_runtime_cgroups }} --kubelet-cgroups={{ kubelet_kubelet_cgroups }} \ {% endset %} {# Node reserved CPU/memory #} diff --git a/scale.yml b/scale.yml index 676fba610..ff027d6c0 100644 --- a/scale.yml +++ b/scale.yml @@ -36,6 +36,7 @@ - { role: kubespray-defaults} - { role: kubernetes/preinstall, tags: preinstall } - { role: docker, tags: docker, when: manage_docker|default(true) } + - { role: cri-o, tags: crio, when: manage_crio } - role: rkt tags: rkt when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]" diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index abc89e18f..4e4cfb654 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -35,6 +35,7 @@ - { role: kubespray-defaults} - { role: kubernetes/preinstall, tags: preinstall } - { role: docker, tags: docker, when: manage_docker|default(true) } + - { role: cri-o, tags: crio, when: manage_crio } - role: rkt tags: rkt when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"