Cleanup fedora coreos with crio container (#5887)

* fix upgrade of crio on fcos
- update documents

* install conntrack required by kube-proxy
- like commit 48c41bcbe7

* enable fedora modular repo for crio

* allow to override crio configuration
- set cgroup manager same to kubelet_cgroup_driver if defined
- path of seccomp_profile depends on distribution

* allow to override crio configuration
- fix path for ubuntu

* allow to override crio configuration
- fix cni path for fcos

README.md

@@ -105,7 +105,7 @@ vagrant up
 - **Ubuntu** 16.04, 18.04
 - **CentOS/RHEL** 7, 8 (experimental: see [centos 8 notes](docs/centos8.md)
 - **Fedora** 28
-- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md)
+- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md))
 - **openSUSE** Leap 42.3/Tumbleweed
 - **Oracle Linux** 7
 

docs/fcos.md

@@ -1,6 +1,7 @@
 # Fedora CoreOS
 
-Tested with stable version 31.20200223.3.0
+Tested with stable version 31.20200223.3.0.
+
 Because package installation with `rpm-ostree` requires a reboot, playbook may fail while bootstrap.
 Restart playbook again.
 
@@ -35,11 +36,25 @@ systemd:
         WantedBy=multi-user.target
 ```
 
+## Network
+
+### calico
+
+To use calico create sysctl file with ignition:
+
+```yaml
+files:
+    - path: /etc/sysctl.d/reverse-path-filter.conf
+      contents:
+        inline: |
+          net.ipv4.conf.all.rp_filter=1
+```
+
 ## libvirt setup
 
 ### Prepare
 
-Prepare ignition and serve via http (a.e. python -m SimpleHTTPServer )
+Prepare ignition and serve via http (a.e. python -m http.server )
 
 ```json
 {
@@ -50,10 +65,9 @@ Prepare ignition and serve via http (a.e. python -m SimpleHTTPServer )
   "passwd": {
     "users": [
       {
-        "name": "adi",
-        "passwordHash": "$1$.RGu8J4x$U7uxcOg/eotTEIRxhk62I0",
+        "name": "ansibleUser",
         "sshAuthorizedKeys": [
-          "ssh-rsa ..fillyouruser"
+          "ssh-rsa ..publickey.."
         ],
         "groups": [ "wheel" ]
       }

roles/bootstrap-os/defaults/main.yml

@@ -19,9 +19,10 @@ fedora_coreos_packages:
   - dbus-tools              # because of networkManager reload bug (https://bugzilla.redhat.com/show_bug.cgi?id=1745659)
   - ethtool                 # required in kubeadm preflight phase for verifying the environment
   - ipset                   # required in kubeadm preflight phase for verifying the environment
+  - conntrack-tools         # required by kube-proxy
 
 ## General
 # Set the hostname to inventory_hostname
 override_system_hostname: true
 
-is_fedora_coreos: false
+is_fedora_coreos: false

roles/container-engine/cri-o/defaults/main.yml

@@ -1,2 +1,8 @@
 ---
 crio_rhel_repo_base_url: 'https://cbs.centos.org/repos/paas7-crio-114-candidate/x86_64/os/'
+
+crio_seccomp_profile: "/etc/crio/seccomp.json"
+
+crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('cgroupfs') }}"
+
+crio_runc_path: "/usr/sbin/runc"

roles/container-engine/cri-o/tasks/main.yaml

@@ -71,15 +71,33 @@
   register: need_bootstrap_crio
   when: is_ostree
 
+- name: Enable modular repos for crio
+  ini_file:
+    path: "/etc/yum.repos.d/{{ item }}.repo"
+    section: "{{ item }}"
+    option: enabled
+    value: 1
+  become: true
+  when:
+    - is_ostree
+    - not need_bootstrap_crio.stat.exists
+  loop:
+    - "fedora-updates-modular"
+    - "fedora-modular"
+
 - name: Install cri-o packages with osttree
-  raw: "export http_proxy={{ http_proxy | default('') }} && rpm-ostree install {{ crio_packages|join(' ') }}"
-  when: is_ostree and not need_bootstrap_crio.stat.exists
+  command: "rpm-ostree install {{ crio_packages|join(' ') }}"
+  when:
+    - is_ostree
+    - not need_bootstrap_crio.stat.exists
   become: true
 
 - name: Reboot immediately for updated ostree
   reboot:
   become: true
-  when: is_ostree and not need_bootstrap_crio.stat.exists
+  when:
+    - is_ostree
+    - not need_bootstrap_crio.stat.exists
 
 - name: Install cri-o config
   template:

roles/container-engine/cri-o/templates/crio.conf.j2

@@ -102,20 +102,14 @@ selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }}
 
 # Path to the seccomp.json profile which is used as the default seccomp profile
 # for the runtime.
-{% if ansible_os_family == "ClearLinux" %}
-seccomp_profile = "/usr/share/defaults/crio/seccomp.json"
-{% elif ansible_distribution == "Ubuntu" or is_fedora_coreos %}
-seccomp_profile = ""
-{% else %}
-seccomp_profile = "/etc/crio/seccomp.json"
-{% endif %}
+seccomp_profile = "{{crio_seccomp_profile}}"
 
 # Used to change the name of the default AppArmor profile of CRI-O. The default
 # profile name is "crio-default-" followed by the version string of CRI-O.
 apparmor_profile = "crio-default"
 
 # Cgroup management implementation used for the runtime.
-cgroup_manager = "cgroupfs"
+cgroup_manager = "{{crio_cgroup_manager}}"
 
 # List of default capabilities for containers. If it is empty or commented out,
 # only the capabilities defined in the containers json file by the user/kube
@@ -218,13 +212,7 @@ ctr_stop_timeout = 0
   # of trust of the workload.
 
   [crio.runtime.runtimes.runc]
-{% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" %}
-  runtime_path = "/usr/bin/runc"
-{% elif ansible_distribution == "Ubuntu" %}
-  runtime_path = "/usr/lib/cri-o-runc/sbin/runc"
-{% else %}
-  runtime_path = "/usr/sbin/runc"
-{% endif %}
+  runtime_path = "{{ crio_runc_path }}"
   runtime_type = "oci"
 
 
@@ -293,7 +281,7 @@ network_dir = "/etc/cni/net.d/"
 # Paths to directories where CNI plugin binaries are located.
 plugin_dirs = [
 	"/usr/libexec/cni",
-{% if ansible_os_family == "ClearLinux" %}
+{% if ansible_os_family == "ClearLinux" or is_ostree %}
 	"/opt/cni/bin/",
 {% endif %}
 ]

roles/container-engine/cri-o/vars/clearlinux.yml

@@ -4,3 +4,5 @@ crio_packages:
 
 crio_service: crio
 crio_conmon: /usr/libexec/crio/conmon
+crio_seccomp_profile: /usr/share/defaults/crio/seccomp.json
+crio_runc_path: /usr/bin/runc

roles/container-engine/cri-o/vars/fedora.yml

@@ -5,3 +5,4 @@ crio_packages:
 
 crio_service: cri-o
 crio_conmon: /usr/libexec/crio/conmon
+crio_seccomp_profile: ""

roles/container-engine/cri-o/vars/redhat.yml

@@ -5,3 +5,4 @@ crio_packages:
 
 crio_service: crio
 crio_conmon: /usr/libexec/crio/conmon
+crio_runc_path: /usr/bin/runc

roles/container-engine/cri-o/vars/ubuntu.yml

@@ -3,4 +3,6 @@ crio_packages:
   - "cri-o-{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
 
 crio_service: crio
-crio_conmon: /usr/bin/conmon
+crio_conmon: /usr/libexec/podman/conmon
+crio_seccomp_profile: ""
+crio_runc_path: /usr/lib/cri-o-runc/sbin/runc