Merge pull request #11131 from VannTen/design/modular_pkgs_install

Fine grained OS packages installation

roles/kubernetes/preinstall/defaults/main.yml

@@ -6,18 +6,6 @@ epel_enabled: false
 # Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
 dns_late: false
 
-common_required_pkgs:
-  - "{{ (ansible_distribution == 'openSUSE Tumbleweed') | ternary('openssl-1_1', 'openssl') }}"
-  - curl
-  - rsync
-  - socat
-  - unzip
-  - e2fsprogs
-  - xfsprogs
-  - ebtables
-  - bash-completion
-  - tar
-
 # Set to true if your network does not support IPv6
 # This may be necessary for pulling Docker images from
 # GCE docker repository

roles/kubernetes/preinstall/files/pkgs-schema.json

@@ -0,0 +1,80 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://kubespray.io/internal/os_packages.schema.json",
+    "title": "Os packages",
+    "description": "Criteria for selecting packages to install on Kubernetes nodes during installation by Kubespray",
+    "type": "object",
+    "patternProperties": {
+        ".*": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "enabled": {
+                    "description": "Escape hatch to filter packages. The value is expected to be pre-resolved to a boolean by Jinja",
+                    "type": "boolean",
+                    "default": true
+                },
+                "groups": {
+                    "description": "Match if the host is in one of these groups. If not specified match any host.",
+                    "type": "array",
+                    "minItems": 1,
+                    "items":{
+                        "type": "string",
+                        "pattern": "^[0-9A-Za-z_]*$"
+                    }
+                },
+                "os": {
+                    "type": "object",
+                    "description": "If not specified match any OS. Otherwise, must match by 'families' or 'distributions' to be included.",
+                    "additionalProperties": false,
+                    "minProperties": 1,
+                    "properties": {
+                        "families": {
+                            "description": "Match if ansible_os_family is part of the list.",
+                            "type": "array",
+                            "minItems": 1,
+                            "items": {
+                                "type": "string"
+                            }
+                        },
+                        "distributions": {
+                            "type": "object",
+                            "description": "Match if ansible_distribution match one of defined keys.",
+                            "minProperties": 1,
+                            "patternProperties": {
+                                ".*": {
+                                    "description": "Match if either the value is the empty hash, or one major_versions/versions/releases contains the corresponding variable ('ansible_distrbution_*')",
+                                    "type": "object",
+                                    "additionalProperties": false,
+                                    "properties": {
+                                        "major_versions": {
+                                            "type": "array",
+                                            "minItems": 1,
+                                            "items": {
+                                                "type": "string"
+                                            }
+                                        },
+                                        "versions": {
+                                            "type": "array",
+                                            "minItems": 1,
+                                            "items": {
+                                                "type": "string"
+                                            }
+                                        },
+                                        "releases": {
+                                            "type": "array",
+                                            "minItems": 1,
+                                            "items": {
+                                                "type": "string"
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

roles/kubernetes/preinstall/tasks/0020-set_facts.yml

@@ -199,20 +199,6 @@
       supersede domain-name-servers {{ (nameservers | d([]) + cloud_resolver | d([])) | unique | join(', ') }};
   when: dns_early and not dns_late
 
-- name: Gather os specific variables
-  include_vars: "{{ item }}"
-  with_first_found:
-    - files:
-        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml"
-        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml"
-        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml"
-        - "{{ ansible_distribution | lower }}.yml"
-        - "{{ ansible_os_family | lower }}.yml"
-        - defaults.yml
-      paths:
-        - ../vars
-      skip: true
-
 - name: Set etcd vars if using kubeadm mode
   set_fact:
     etcd_cert_dir: "{{ kube_cert_dir }}"

roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -316,3 +316,15 @@
   when:
     - kube_apiserver_enable_admission_plugins is defined
     - kube_apiserver_enable_admission_plugins | length > 0
+
+- name: Verify that the packages list structure is valid
+  ansible.utils.validate:
+    criteria: "{{ lookup('file', 'pkgs-schema.json') }}"
+    data: "{{ pkgs }}"
+
+- name: Verify that the packages list is sorted
+  vars:
+    pkgs_lists: "{{ pkgs.keys() | list }}"
+  assert:
+    that: "pkgs_lists | sort == pkgs_lists"
+    fail_msg: "pkgs is not sorted: {{ pkgs_lists | ansible.utils.fact_diff(pkgs_lists | sort) }}"

roles/kubernetes/preinstall/tasks/0070-system-packages.yml

@@ -59,19 +59,28 @@
   tags:
     - bootstrap-os
 
-- name: Update common_required_pkgs with ipvsadm when kube_proxy_mode is ipvs
-  set_fact:
-    common_required_pkgs: "{{ common_required_pkgs | default([]) + ['ipvsadm', 'ipset'] }}"
-  when: kube_proxy_mode == 'ipvs'
-
 - name: Install packages requirements
+  vars:
+    # The json_query for selecting packages name is split for readability
+    # see files/pkgs-schema.json for the structure of `pkgs`
+    # and the matching semantics
+    full_query: "[? value | (enabled == null || enabled) && ( {{ filters_os }} ) && ( {{ filters_groups }} ) ].key"
+    filters_groups: "groups | @ == null || [? contains(`{{ group_names }}`, @)]"
+    filters_os: "os == null || (os | ( {{ filters_family }} ) || ( {{ filters_distro }} ))"
+    dquote: !unsafe '"'
+    # necessary to workaround Ansible escaping
+    filters_distro: "distributions.{{ dquote }}{{ ansible_distribution  }}{{ dquote }} |
+                          @ == `{}` ||
+                          contains(not_null(major_versions, `[]`), '{{ ansible_distribution_major_version }}') ||
+                          contains(not_null(versions, `[]`), '{{ ansible_distribution_version }}') ||
+                          contains(not_null(releases, `[]`), '{{ ansible_distribution_release }}')"
+    filters_family: "families && contains(families, '{{ ansible_os_family }}')"
   package:
-    name: "{{ required_pkgs | default([]) | union(common_required_pkgs | default([])) }}"
+    name: "{{ pkgs | dict2items | to_json|from_json | community.general.json_query(full_query) }}"
     state: present
   register: pkgs_task_result
   until: pkgs_task_result is succeeded
   retries: "{{ pkg_install_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
-  when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] or is_fedora_coreos)
   tags:
     - bootstrap-os

roles/kubernetes/preinstall/vars/amazon.yml

@@ -1,7 +0,0 @@
----
-required_pkgs:
-  - libselinux-python
-  - device-mapper-libs
-  - nss
-  - conntrack-tools
-  - libseccomp

roles/kubernetes/preinstall/vars/centos.yml

@@ -1,8 +0,0 @@
----
-required_pkgs:
-  - "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}"
-  - device-mapper-libs
-  - nss
-  - conntrack
-  - container-selinux
-  - libseccomp

roles/kubernetes/preinstall/vars/debian-11.yml

@@ -1,10 +0,0 @@
----
-required_pkgs:
-  - python3-apt
-  - gnupg
-  - apt-transport-https
-  - software-properties-common
-  - conntrack
-  - iptables
-  - apparmor
-  - libseccomp2

roles/kubernetes/preinstall/vars/debian-12.yml

@@ -1,11 +0,0 @@
----
-required_pkgs:
-  - python3-apt
-  - gnupg
-  - apt-transport-https
-  - software-properties-common
-  - conntrack
-  - iptables
-  - apparmor
-  - libseccomp2
-  - mergerfs

roles/kubernetes/preinstall/vars/debian.yml

@@ -1,9 +0,0 @@
----
-required_pkgs:
-  - python-apt
-  - aufs-tools
-  - apt-transport-https
-  - software-properties-common
-  - conntrack
-  - apparmor
-  - libseccomp2

roles/kubernetes/preinstall/vars/fedora.yml

@@ -1,8 +0,0 @@
----
-required_pkgs:
-  - iptables
-  - libselinux-python3
-  - device-mapper-libs
-  - conntrack
-  - container-selinux
-  - libseccomp

roles/kubernetes/preinstall/vars/main.yml

@@ -0,0 +1,106 @@
+---
+pkgs:
+  apparmor: &debian_family_base
+    os:
+      families:
+      - Debian
+  apt-transport-https: *debian_family_base
+  aufs-tools: &deb_10
+    groups:
+    - k8s_cluster
+    os:
+      distributions:
+        Debian:
+          major_versions:
+          - "10"
+  bash-completion: {}
+  conntrack: &deb_redhat
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - Debian
+      - RedHat
+  conntrack-tools:
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - Suse
+      distributions:
+        Amazon: {}
+  container-selinux: &redhat_family
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - RedHat
+  curl: {}
+  device-mapper:
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - Suse
+  device-mapper-libs: *redhat_family
+  e2fsprogs: {}
+  ebtables: {}
+  gnupg: &debian
+    groups:
+    - k8s_cluster
+    os:
+      distributions:
+        Debian:
+          major_versions:
+          - "11"
+          - "12"
+  ipset:
+    enabled: "{{ kube_proxy_mode != 'ipvs' }}"
+    groups:
+    - k8s_cluster
+  iptables: *deb_redhat
+  ipvsadm:
+    enabled: "{{ kube_proxy_mode == 'ipvs' }}"
+    groups:
+    - k8s_cluster
+  libseccomp: *redhat_family
+  libseccomp2:
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - Suse
+      - Debian
+  libselinux-python:  # TODO: Handle rehat_family + major < 8
+    os:
+      distributions:
+        Amazon: {}
+  libselinux-python3:
+    os:
+      distributions:
+        Fedora: {}
+  mergerfs:
+    os:
+      distributions:
+        Debian:
+          major_versions:
+          - "12"
+  nss: *redhat_family
+  openssl: {}
+  python-apt: *deb_10
+  # TODO: not for debian 10
+  python3-apt: *debian_family_base
+  python3-libselinux:
+    os:
+      distributions:
+        RedHat: &major_redhat_like
+          major_versions:
+          - "8"
+          - "9"
+        Centos: *major_redhat_like
+  rsync: {}
+  socat: {}
+  software-properties-common: *debian_family_base
+  tar: {}
+  unzip: {}
+  xfsprogs: {}

roles/kubernetes/preinstall/vars/redhat.yml

@@ -1,8 +0,0 @@
----
-required_pkgs:
-  - "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}"
-  - device-mapper-libs
-  - nss
-  - conntrack
-  - container-selinux
-  - libseccomp

roles/kubernetes/preinstall/vars/suse.yml

@@ -1,5 +0,0 @@
----
-required_pkgs:
-  - device-mapper
-  - conntrack-tools
-  - libseccomp2

roles/kubernetes/preinstall/vars/ubuntu.yml

@@ -1,8 +0,0 @@
----
-required_pkgs:
-  - python3-apt
-  - apt-transport-https
-  - software-properties-common
-  - conntrack
-  - apparmor
-  - libseccomp2