Add cri-o role.

cluster.yml

@@ -34,6 +34,7 @@
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker, when: manage_docker|default(true) }
+    - { role: cri-o, tags: crio, when: manage_crio }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"

roles/cri-o/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+crio_rhel_repo_base_url: 'https://cbs.centos.org/repos/paas7-openshift-origin311-candidate/x86_64/os/'

roles/cri-o/tasks/main.yaml

@@ -0,0 +1,40 @@
+---
+- name: Add OpenShift Origin repository
+  yum_repository:
+    name: origin
+    description: OpenShift Origin Repo
+    baseurl: "{{ crio_rhel_repo_base_url }}"
+    gpgcheck: no
+  when:
+    - ansible_os_family == 'RedHat'
+
+- name: Install cri-o
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - cri-o
+    - cri-tools
+    - oci-systemd-hook
+
+- name: Install cri-o config
+  template:
+    src: crio.conf.j2
+    dest: /etc/crio/crio.conf
+
+- name: Copy mounts.conf
+  shell: |
+    cp -T /usr/share/containers/mounts.conf /etc/containers/mounts.conf
+
+- name: Create directory for oci hooks
+  file:
+    path: /etc/containers/oci/hooks.d
+    state: directory
+    owner: root
+    mode: 0755
+
+- name: Install cri-o service
+  service:
+    name: crio
+    enabled: yes
+    state: restarted

roles/cri-o/templates/crio.conf.j2

@@ -0,0 +1,234 @@
+
+# The "crio" table contains all of the server options.
+[crio]
+
+# CRI-O reads its storage defaults from the containers/storage configuration
+# file, /etc/containers/storage.conf. Modify storage.conf if you want to
+# change default storage for all tools that use containers/storage.  If you
+# want to modify just crio, you can change the storage configuration in this
+# file.
+
+# root is a path to the "root directory". CRIO stores all of its data,
+# including container images, in this directory.
+#root = "/var/lib/containers/storage"
+
+# run is a path to the "run directory". CRIO stores all of its state
+# in this directory.
+#runroot = "/var/run/containers/storage"
+
+# storage_driver select which storage driver is used to manage storage
+# of images and containers.
+storage_driver = "overlay2"
+
+# storage_option is used to pass an option to the storage driver.
+#storage_option = [
+#]
+
+# The "crio.api" table contains settings for the kubelet/gRPC interface.
+[crio.api]
+
+# listen is the path to the AF_LOCAL socket on which crio will listen.
+listen = "/var/run/crio/crio.sock"
+
+# stream_address is the IP address on which the stream server will listen
+stream_address = ""
+
+# stream_port is the port on which the stream server will listen
+stream_port = "10010"
+
+# stream_enable_tls enables encrypted tls transport of the stream server
+stream_enable_tls = false
+
+# stream_tls_cert is the x509 certificate file path used to serve the encrypted stream.
+# This file can change, and CRIO will automatically pick up the changes within 5 minutes.
+stream_tls_cert = ""
+
+# stream_tls_key is the key file path used to serve the encrypted stream.
+# This file can change, and CRIO will automatically pick up the changes within 5 minutes.
+stream_tls_key = ""
+
+# stream_tls_ca is the x509 CA(s) file used to verify and authenticate client
+# communication with the tls encrypted stream.
+# This file can change, and CRIO will automatically pick up the changes within 5 minutes.
+stream_tls_ca = ""
+
+# file_locking is whether file-based locking will be used instead of
+# in-memory locking
+file_locking = true
+
+# The "crio.runtime" table contains settings pertaining to the OCI
+# runtime used and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# runtime is the OCI compatible runtime used for trusted container workloads.
+# This is a mandatory setting as this runtime will be the default one
+# and will also be used for untrusted container workloads if
+# runtime_untrusted_workload is not set.
+runtime = "/usr/bin/runc"
+
+# runtime_untrusted_workload is the OCI compatible runtime used for untrusted
+# container workloads. This is an optional setting, except if
+# default_container_trust is set to "untrusted".
+runtime_untrusted_workload = ""
+
+# default_workload_trust is the default level of trust crio puts in container
+# workloads. It can either be "trusted" or "untrusted", and the default
+# is "trusted".
+# Containers can be run through different container runtimes, depending on
+# the trust hints we receive from kubelet:
+# - If kubelet tags a container workload as untrusted, crio will try first to
+# run it through the untrusted container workload runtime. If it is not set,
+# crio will use the trusted runtime.
+# - If kubelet does not provide any information about the container workload trust
+# level, the selected runtime will depend on the default_container_trust setting.
+# If it is set to "untrusted", then all containers except for the host privileged
+# ones, will be run by the runtime_untrusted_workload runtime. Host privileged
+# containers are by definition trusted and will always use the trusted container
+# runtime. If default_container_trust is set to "trusted", crio will use the trusted
+# container runtime for all containers.
+default_workload_trust = "trusted"
+
+# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE
+no_pivot = false
+
+# conmon is the path to conmon binary, used for managing the runtime.
+conmon = "/usr/libexec/crio/conmon"
+
+# conmon_env is the environment variable list for conmon process,
+# used for passing necessary environment variable to conmon or runtime.
+conmon_env = [
+	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+]
+
+# selinux indicates whether or not SELinux will be used for pod
+# separation on the host. If you enable this flag, SELinux must be running
+# on the host.
+selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }}
+
+# seccomp_profile is the seccomp json profile path which is used as the
+# default for the runtime.
+seccomp_profile = "/etc/crio/seccomp.json"
+
+# apparmor_profile is the apparmor profile name which is used as the
+# default for the runtime.
+apparmor_profile = "crio-default"
+
+# cgroup_manager is the cgroup management implementation to be used
+# for the runtime.
+cgroup_manager = "cgroupfs"
+
+# default_capabilities is the list of capabilities to add and can be modified here.
+# If capabilities below is commented out, the default list of capabilities defined in the
+# spec will be added.
+# If capabilities is empty below, only the capabilities defined in the container json
+# file by the user/kube will be added.
+default_capabilities = [
+	"CHOWN", 
+	"DAC_OVERRIDE", 
+	"FSETID", 
+	"FOWNER", 
+	"NET_RAW", 
+	"SETGID", 
+	"SETUID", 
+	"SETPCAP", 
+	"NET_BIND_SERVICE", 
+	"SYS_CHROOT", 
+	"KILL", 
+]
+
+# hooks_dir_path is the oci hooks directory for automatically executed hooks
+hooks_dir_path = "/usr/share/containers/oci/hooks.d"
+
+# default_mounts is the mounts list to be mounted for the container when created
+# deprecated, will be taken out in future versions, add default mounts to either
+# /usr/share/containers/mounts.conf or /etc/containers/mounts.conf
+default_mounts = [
+]
+
+# CRI-O reads its default mounts from the following two files:
+# 1) /etc/containers/mounts.conf - this is the override file, where users can
+# either add in their own default mounts, or override the default mounts shipped
+# with the package.
+# 2) /usr/share/containers/mounts.conf - this is the default file read for mounts.
+# If you want CRI-O to read from a different, specific mounts file, you can change
+# the default_mounts_file path right below. Note, if this is done, CRI-O will only add
+# mounts it finds in this file.
+
+# default_mounts_file is the file path holding the default mounts to be mounted for the
+# container when created.
+# default_mounts_file = ""
+
+# pids_limit is the number of processes allowed in a container
+pids_limit = 1024
+
+# log_size_max is the max limit for the container log size in bytes.
+# Negative values indicate that no limit is imposed.
+log_size_max = -1
+
+# read-only indicates whether all containers will run in read-only mode
+read_only = false
+
+# The "crio.image" table contains settings pertaining to the
+# management of OCI images.
+
+# uid_mappings specifies the UID mappings to have in the user namespace.
+# A range is specified in the form containerUID:HostUID:Size.  Multiple
+# ranges are separed by comma.
+uid_mappings = ""
+
+# gid_mappings specifies the GID mappings to have in the user namespace.
+# A range is specified in the form containerGID:HostGID:Size.  Multiple
+# ranges are separed by comma.
+gid_mappings = ""
+
+[crio.image]
+
+# default_transport is the prefix we try prepending to an image name if the
+# image name as we receive it can't be parsed as a valid source reference
+default_transport = "docker://"
+
+# pause_image is the image which we use to instantiate infra containers.
+pause_image = "docker://k8s.gcr.io/pause:3.1"
+
+# pause_command is the command to run in a pause_image to have a container just
+# sit there.  If the image contains the necessary information, this value need
+# not be specified.
+pause_command = "/pause"
+
+# signature_policy is the name of the file which decides what sort of policy we
+# use when deciding whether or not to trust an image that we've pulled.
+# Outside of testing situations, it is strongly advised that this be left
+# unspecified so that the default system-wide policy will be used.
+signature_policy = ""
+
+# image_volumes controls how image volumes are handled.
+# The valid values are mkdir and ignore.
+image_volumes = "mkdir"
+
+# CRI-O reads its configured registries defaults from the containers/image configuration
+# file, /etc/containers/registries.conf. Modify registries.conf if you want to
+# change default registries for all tools that use containers/image.  If you
+# want to modify just crio, you can change the registies configuration in this
+# file.
+
+# insecure_registries is used to skip TLS verification when pulling images.
+insecure_registries = [
+  "{{ kube_service_addresses }}"
+]
+
+# registries is used to specify a comma separated list of registries to be used
+# when pulling an unqualified image (e.g. fedora:rawhide).
+registries = [
+  "docker.io"
+]
+
+# The "crio.network" table contains settings pertaining to the
+# management of CNI plugins.
+[crio.network]
+
+# network_dir is is where CNI network configuration
+# files are stored.
+network_dir = "/etc/cni/net.d/"
+
+# plugin_dir is is where CNI plugin binaries are stored.
+plugin_dir = "/opt/cni/bin/"

roles/kubernetes/node/defaults/main.yml

@@ -31,6 +31,11 @@ kubelet_cgroups_per_qos: true
 # Set to empty to avoid cgroup creation
 kubelet_enforce_node_allocatable: "\"\""
 
+# Set runtime cgroups
+kubelet_runtime_cgroups: "/systemd/system.slice"
+# Set kubelet cgroups
+kubelet_kubelet_cgroups: "/systemd/system.slice"
+
 # Set false to enable sharing a pid namespace between containers in a pod.
 # Note that PID namespace sharing requires docker >= 1.13.1.
 kubelet_disable_shared_pid: true

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -42,6 +42,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% else %}
 --fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \
 {% endif %}
+--runtime-cgroups={{ kubelet_runtime_cgroups }} --kubelet-cgroups={{ kubelet_kubelet_cgroups }} \
 {% endset %}
 
 {# Node reserved CPU/memory #}

scale.yml

@@ -36,6 +36,7 @@
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker, when: manage_docker|default(true) }
+    - { role: cri-o, tags: crio, when: manage_crio }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"

upgrade-cluster.yml

@@ -35,6 +35,7 @@
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker, when: manage_docker|default(true) }
+    - { role: cri-o, tags: crio, when: manage_crio }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"