project: upgrade test dependencies and drop ansible-core 2.11 (#10034)

Molecule 5.0 require ansible-core 2.12.10.
So this commit we update ansible-core from 2.12.5 to 2.12.10.
We also drop supporting two ansible-core version. Also we now use the "oldest"
still supported ansible-core version as both 2.11 is EOL and not
supported by molecule.



tests/molecule: remove linting in molecule to support molecule 5



tests/molecule: remove role name check for molecule 5 support

Kubespray doesn't use ansible galaxy style naming so we have to disable
that check.



contrib/inventory_builder: fix tox.ini for tox4



tests/molecule: fix get_playbook in testinfra tests



tests: upgrade most tests requirements

Exclude ansible-lint for now, I will do that in a separate PR.



tests/molecule: force kvm driver option

If we don't do this it fallbacks to qemu emulated on our CI for some
reasons.

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

.gitlab-ci.yml

@@ -35,14 +35,13 @@ variables:
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
   TERRAFORM_VERSION: 1.3.7
-  ANSIBLE_MAJOR_VERSION: "2.11"
   PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
 
 before_script:
   - ./tests/scripts/rebase.sh
   - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
   - python -m pip uninstall -y ansible ansible-base ansible-core
-  - python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt
+  - python -m pip install -r tests/requirements.txt
   - mkdir -p /.ssh
 
 .job: &job

.gitlab-ci/packet.yml

@@ -31,14 +31,6 @@ packet_ubuntu20-calico-aio:
   variables:
     RESET_CHECK: "true"
 
-packet_ubuntu20-calico-aio-ansible-2_11:
-  stage: deploy-part1
-  extends: .packet_periodic
-  when: on_success
-  variables:
-    ANSIBLE_MAJOR_VERSION: "2.11"
-    RESET_CHECK: "true"
-
 # ### PR JOBS PART2
 
 packet_ubuntu20-aio-docker:

Dockerfile

@@ -2,7 +2,7 @@
 FROM ubuntu:jammy-20230308
 # Some tools like yamllint need this
 # Pip needs this as well at the moment to install ansible
-# (and potentially other packages) 
+# (and potentially other packages)
 # See: https://github.com/pypa/pip/issues/10219
 ENV LANG=C.UTF-8 \
     DEBIAN_FRONTEND=noninteractive \
@@ -29,7 +29,7 @@ RUN apt update -q \
        openssh-client \
     && pip install --no-compile --no-cache-dir \
        ansible==5.7.1 \
-       ansible-core==2.12.5 \
+       ansible-core==2.12.10 \
        cryptography==3.4.8 \
        jinja2==3.1.2 \
        netaddr==0.8.0 \

contrib/inventory_builder/tox.ini

@@ -1,21 +1,27 @@
 [tox]
 minversion = 1.6
 skipsdist = True
-envlist = pep8, py33
+envlist = pep8
 
 [testenv]
-whitelist_externals = py.test
+allowlist_externals = py.test
 usedevelop = True
 deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/test-requirements.txt
 setenv = VIRTUAL_ENV={envdir}
-passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+passenv =
+    http_proxy
+    HTTP_PROXY
+    https_proxy
+    HTTPS_PROXY
+    no_proxy
+    NO_PROXY
 commands = pytest -vv #{posargs:./tests}
 
 [testenv:pep8]
 usedevelop = False
-whitelist_externals = bash
+allowlist_externals = bash
 commands =
     bash -c "find {toxinidir}/* -type f -name '*.py' -print0 | xargs -0 flake8"
 

pipeline.Dockerfile

@@ -37,12 +37,12 @@ RUN apt update -q \
 
 WORKDIR /kubespray
 
-RUN --mount=type=bind,target=./requirements-2.12.txt,src=./requirements-2.12.txt \
-    --mount=type=bind,target=./tests/requirements-2.12.txt,src=./tests/requirements-2.12.txt \
+RUN --mount=type=bind,target=./requirements.txt,src=./requirements.txt \
+    --mount=type=bind,target=./tests/requirements.txt,src=./tests/requirements.txt \
     --mount=type=bind,target=./roles/kubespray-defaults/defaults/main.yaml,src=./roles/kubespray-defaults/defaults/main.yaml \
     update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
-    && pip install --no-compile --no-cache-dir -r tests/requirements-2.12.txt \
+    && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \
     && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
     && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \

playbooks/ansible_version.yml

@@ -3,7 +3,7 @@
   gather_facts: false
   become: no
   vars:
-    minimal_ansible_version: 2.11.0
+    minimal_ansible_version: 2.12.0
     maximal_ansible_version: 2.13.0
     ansible_connection: local
   tags: always

requirements-2.11.txt

@@ -1,10 +0,0 @@
-ansible==4.10.0
-ansible-core==2.11.11
-cryptography==3.4.8
-jinja2==3.1.2
-jmespath==1.0.1
-MarkupSafe==2.1.2
-netaddr==0.8.0
-pbr==5.11.1
-ruamel.yaml==0.17.21
-ruamel.yaml.clib==0.2.7

requirements-2.12.txt

@@ -1,10 +0,0 @@
-ansible==5.7.1
-ansible-core==2.12.5
-cryptography==3.4.8
-jinja2==3.1.2
-jmespath==1.0.1
-MarkupSafe==2.1.2
-netaddr==0.8.0
-pbr==5.11.1
-ruamel.yaml==0.17.21
-ruamel.yaml.clib==0.2.7

requirements.txt

@@ -1 +0,0 @@
-requirements-2.12.txt

requirements.txt

@@ -0,0 +1,10 @@
+ansible==5.7.1
+ansible-core==2.12.10
+cryptography==3.4.8
+jinja2==3.1.2
+jmespath==1.0.1
+MarkupSafe==2.1.2
+netaddr==0.8.0
+pbr==5.11.1
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.7

roles/adduser/molecule/default/molecule.yml

@@ -1,9 +1,7 @@
 ---
+role_name_check: 1
 dependency:
   name: galaxy
-lint: |
-  set -e
-  yamllint -c ../../.yamllint .
 driver:
   name: vagrant
   provider:
@@ -13,15 +11,13 @@ platforms:
     box: generic/ubuntu2004
     cpus: 1
     memory: 512
+    provider_options:
+      driver: kvm
 provisioner:
   name: ansible
   config_options:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
-  lint:
-    name: ansible-lint
 verifier:
   name: testinfra
-  lint:
-    name: flake8

roles/adduser/molecule/default/tests/test_default.py

@@ -1,12 +1,15 @@
 import os
-import yaml
-import glob
+from pathlib import Path
+
 import testinfra.utils.ansible_runner
-from ansible.playbook import Playbook
+import yaml
 from ansible.cli.playbook import PlaybookCLI
+from ansible.playbook import Playbook
 
 testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+    os.environ["MOLECULE_INVENTORY_FILE"]
+).get_hosts("all")
+
 
 def read_playbook(playbook):
     cli_args = [os.path.realpath(playbook), testinfra_hosts]
@@ -19,19 +22,22 @@ def read_playbook(playbook):
     for play in pb.get_plays():
         yield variable_manager.get_vars(play)
 
+
 def get_playbook():
-    with open(os.path.realpath(' '.join(map(str,glob.glob('molecule.*')))), 'r') as yamlfile:
+    playbooks_path = Path(__file__).parent.parent
+    with open(os.path.join(playbooks_path, "molecule.yml"), "r") as yamlfile:
         data = yaml.load(yamlfile, Loader=yaml.FullLoader)
-        if 'playbooks' in data['provisioner'].keys():
-            if 'converge' in data['provisioner']['playbooks'].keys():
-                return data['provisioner']['playbooks']['converge']
+        if "playbooks" in data["provisioner"].keys():
+            if "converge" in data["provisioner"]["playbooks"].keys():
+                return data["provisioner"]["playbooks"]["converge"]
         else:
-            return ' '.join(map(str,glob.glob('converge.*')))
+            return os.path.join(playbooks_path, "converge.yml")
+
 
 def test_user(host):
     for vars in read_playbook(get_playbook()):
-        assert host.user(vars['user']['name']).exists
-        if 'group' in vars['user'].keys():
-            assert host.group(vars['user']['group']).exists
+        assert host.user(vars["user"]["name"]).exists
+        if "group" in vars["user"].keys():
+            assert host.group(vars["user"]["group"]).exists
         else:
-            assert host.group(vars['user']['name']).exists
+            assert host.group(vars["user"]["name"]).exists

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -1,9 +1,7 @@
 ---
+role_name_check: 1
 dependency:
   name: galaxy
-lint: |
-  set -e
-  yamllint -c ../../.yamllint .
 driver:
   name: vagrant
   provider:
@@ -13,14 +11,14 @@ platforms:
     box: generic/ubuntu2004
     cpus: 1
     memory: 512
+    provider_options:
+      driver: kvm
 provisioner:
   name: ansible
   config_options:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
-  lint:
-    name: ansible-lint
   inventory:
     hosts:
       all:
@@ -31,5 +29,3 @@ provisioner:
               bastion-01:
 verifier:
   name: testinfra
-  lint:
-    name: flake8

roles/bastion-ssh-config/molecule/default/tests/test_default.py

@@ -1,12 +1,15 @@
 import os
-import yaml
-import glob
+from pathlib import Path
+
 import testinfra.utils.ansible_runner
-from ansible.playbook import Playbook
+import yaml
 from ansible.cli.playbook import PlaybookCLI
+from ansible.playbook import Playbook
 
 testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+    os.environ["MOLECULE_INVENTORY_FILE"]
+).get_hosts("all")
+
 
 def read_playbook(playbook):
     cli_args = [os.path.realpath(playbook), testinfra_hosts]
@@ -19,16 +22,19 @@ def read_playbook(playbook):
     for play in pb.get_plays():
         yield variable_manager.get_vars(play)
 
+
 def get_playbook():
-    with open(os.path.realpath(' '.join(map(str,glob.glob('molecule.*')))), 'r') as yamlfile:
+    playbooks_path = Path(__file__).parent.parent
+    with open(os.path.join(playbooks_path, "molecule.yml"), "r") as yamlfile:
         data = yaml.load(yamlfile, Loader=yaml.FullLoader)
-        if 'playbooks' in data['provisioner'].keys():
-            if 'converge' in data['provisioner']['playbooks'].keys():
-                return data['provisioner']['playbooks']['converge']
+        if "playbooks" in data["provisioner"].keys():
+            if "converge" in data["provisioner"]["playbooks"].keys():
+                return data["provisioner"]["playbooks"]["converge"]
         else:
-            return ' '.join(map(str,glob.glob('converge.*')))
+            return os.path.join(playbooks_path, "converge.yml")
+
 
 def test_ssh_config(host):
     for vars in read_playbook(get_playbook()):
-        assert host.file(vars['ssh_bastion_confing__name']).exists
-        assert host.file(vars['ssh_bastion_confing__name']).is_file
+        assert host.file(vars["ssh_bastion_confing__name"]).exists
+        assert host.file(vars["ssh_bastion_confing__name"]).is_file

roles/bootstrap-os/molecule/default/molecule.yml

@@ -1,9 +1,7 @@
 ---
+role_name_check: 1
 dependency:
   name: galaxy
-lint: |
-  set -e
-  yamllint -c ../../.yamllint .
 driver:
   name: vagrant
   provider:
@@ -13,30 +11,38 @@ platforms:
     box: generic/ubuntu2004
     cpus: 1
     memory: 512
+    provider_options:
+      driver: kvm
   - name: ubuntu22
     box: generic/ubuntu2204
     cpus: 1
     memory: 1024
+    provider_options:
+      driver: kvm
   - name: centos7
     box: centos/7
     cpus: 1
     memory: 512
+    provider_options:
+      driver: kvm
   - name: almalinux8
     box: almalinux/8
     cpus: 1
     memory: 512
+    provider_options:
+      driver: kvm
   - name: debian10
     box: generic/debian10
     cpus: 1
     memory: 512
+    provider_options:
+      driver: kvm
 provisioner:
   name: ansible
   config_options:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
-  lint:
-    name: ansible-lint
   inventory:
     group_vars:
       all:
@@ -45,5 +51,3 @@ provisioner:
           comment: My test comment
 verifier:
   name: testinfra
-  lint:
-    name: flake8

roles/container-engine/containerd/molecule/default/molecule.yml

@@ -1,11 +1,9 @@
 ---
+role_name_check: 1
 driver:
   name: vagrant
   provider:
     name: libvirt
-lint: |
-  set -e
-  yamllint -c ../../../.yamllint .
 platforms:
   - name: ubuntu20
     box: generic/ubuntu2004
@@ -15,6 +13,8 @@ platforms:
       - kube_control_plane
       - kube_node
       - k8s_cluster
+    provider_options:
+      driver: kvm
   - name: debian11
     box: generic/debian11
     cpus: 1
@@ -23,6 +23,8 @@ platforms:
       - kube_control_plane
       - kube_node
       - k8s_cluster
+    provider_options:
+      driver: kvm
   - name: almalinux8
     box: almalinux/8
     cpus: 1
@@ -31,6 +33,8 @@ platforms:
       - kube_control_plane
       - kube_node
       - k8s_cluster
+    provider_options:
+      driver: kvm
 provisioner:
   name: ansible
   env:
@@ -39,11 +43,5 @@ provisioner:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
-  lint:
-    name: ansible-lint
-    options:
-      c: ../../../.ansible-lint
 verifier:
   name: testinfra
-  lint:
-    name: flake8

roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -1,13 +1,9 @@
 ---
+role_name_check: 1
 driver:
   name: vagrant
   provider:
     name: libvirt
-    options:
-      driver: kvm
-lint: |
-  set -e
-  yamllint -c ../../../.yamllint .
 platforms:
   - name: almalinux8
     box: almalinux/8
@@ -16,6 +12,8 @@ platforms:
     nested: true
     groups:
       - kube_control_plane
+    provider_options:
+      driver: kvm
   - name: ubuntu20
     box: generic/ubuntu2004
     cpus: 1
@@ -23,6 +21,8 @@ platforms:
     nested: true
     groups:
       - kube_control_plane
+    provider_options:
+      driver: kvm
 provisioner:
   name: ansible
   env:
@@ -31,15 +31,9 @@ provisioner:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
-  lint:
-    name: ansible-lint
-    options:
-      c: ../../../.ansible-lint
   inventory:
     group_vars:
       all:
         become: true
 verifier:
   name: testinfra
-  lint:
-    name: flake8

roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -1,11 +1,9 @@
 ---
+role_name_check: 1
 driver:
   name: vagrant
   provider:
     name: libvirt
-lint: |
-  set -e
-  yamllint -c ../../../.yamllint .
 platforms:
   - name: ubuntu20
     box: generic/ubuntu2004
@@ -15,6 +13,8 @@ platforms:
       - kube_control_plane
       - kube_node
       - k8s_cluster
+    provider_options:
+      driver: kvm
   - name: almalinux8
     box: almalinux/8
     cpus: 2
@@ -23,6 +23,8 @@ platforms:
       - kube_control_plane
       - kube_node
       - k8s_cluster
+    provider_options:
+      driver: kvm
   - name: fedora
     box: fedora/36-cloud-base
     cpus: 2
@@ -31,6 +33,8 @@ platforms:
       - kube_control_plane
       - kube_node
       - k8s_cluster
+    provider_options:
+      driver: kvm
   - name: debian10
     box: generic/debian10
     cpus: 2
@@ -39,6 +43,8 @@ platforms:
       - kube_control_plane
       - kube_node
       - k8s_cluster
+    provider_options:
+      driver: kvm
 provisioner:
   name: ansible
   env:
@@ -47,11 +53,5 @@ provisioner:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
-  lint:
-    name: ansible-lint
-    options:
-      c: ../../../.ansible-lint
 verifier:
   name: testinfra
-  lint:
-    name: flake8

roles/container-engine/gvisor/molecule/default/molecule.yml

@@ -1,13 +1,9 @@
 ---
+role_name_check: 1
 driver:
   name: vagrant
   provider:
     name: libvirt
-    options:
-      driver: kvm
-lint: |
-  set -e
-  yamllint -c ../../../.yamllint .
 platforms:
   - name: ubuntu20
     box: generic/ubuntu2004
@@ -16,6 +12,8 @@ platforms:
     nested: true
     groups:
       - kube_control_plane
+    provider_options:
+      driver: kvm
   - name: almalinux8
     box: almalinux/8
     cpus: 1
@@ -23,6 +21,8 @@ platforms:
     nested: true
     groups:
       - kube_control_plane
+    provider_options:
+      driver: kvm
 provisioner:
   name: ansible
   env:
@@ -31,15 +31,9 @@ provisioner:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
-  lint:
-    name: ansible-lint
-    options:
-      c: ../../../.ansible-lint
   inventory:
     group_vars:
       all:
         become: true
 verifier:
   name: testinfra
-  lint:
-    name: flake8

roles/container-engine/kata-containers/molecule/default/molecule.yml

@@ -1,13 +1,9 @@
 ---
+role_name_check: 1
 driver:
   name: vagrant
   provider:
     name: libvirt
-    options:
-      driver: kvm
-lint: |
-  set -e
-  yamllint -c ../../../.yamllint .
 platforms:
   - name: ubuntu20
     box: generic/ubuntu2004
@@ -16,6 +12,8 @@ platforms:
     nested: true
     groups:
       - kube_control_plane
+    provider_options:
+      driver: kvm
   - name: ubuntu22
     box: generic/ubuntu2204
     cpus: 1
@@ -23,6 +21,8 @@ platforms:
     nested: true
     groups:
       - kube_control_plane
+    provider_options:
+      driver: kvm
 provisioner:
   name: ansible
   env:
@@ -31,15 +31,9 @@ provisioner:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
-  lint:
-    name: ansible-lint
-    options:
-      c: ../../../.ansible-lint
   inventory:
     group_vars:
       all:
         become: true
 verifier:
   name: testinfra
-  lint:
-    name: flake8

roles/container-engine/youki/molecule/default/molecule.yml

@@ -1,13 +1,9 @@
 ---
+role_name_check: 1
 driver:
   name: vagrant
   provider:
     name: libvirt
-    options:
-      driver: kvm
-lint: |
-  set -e
-  yamllint -c ../../../.yamllint .
 platforms:
   - name: ubuntu20
     box: generic/ubuntu2004
@@ -16,6 +12,8 @@ platforms:
     nested: true
     groups:
       - kube_control_plane
+    provider_options:
+      driver: kvm
   - name: almalinux8
     box: almalinux/8
     cpus: 1
@@ -23,6 +21,8 @@ platforms:
     nested: true
     groups:
       - kube_control_plane
+    provider_options:
+      driver: kvm
 provisioner:
   name: ansible
   env:
@@ -31,15 +31,9 @@ provisioner:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
-  lint:
-    name: ansible-lint
-    options:
-      c: ../../../.ansible-lint
   inventory:
     group_vars:
       all:
         become: true
 verifier:
   name: testinfra
-  lint:
-    name: flake8

tests/requirements-2.11.txt

@@ -1,11 +0,0 @@
--r ../requirements-2.11.txt
-ansible-lint==5.4.0
-apache-libcloud==2.2.1
-ara[server]==1.6.1
-dopy==0.3.7
-molecule==3.0.6
-molecule-vagrant==0.3
-python-vagrant==0.5.15
-testinfra==5.2.2
-tox==3.11.1
-yamllint==1.19.0

tests/requirements-2.12.txt

@@ -1,11 +0,0 @@
--r ../requirements-2.12.txt
-ansible-lint==5.4.0
-apache-libcloud==2.2.1
-ara[server]==1.6.1
-dopy==0.3.7
-molecule==3.0.6
-molecule-vagrant==0.3
-python-vagrant==0.5.15
-testinfra==5.2.2
-tox==3.11.1
-yamllint==1.19.0

tests/requirements.txt

@@ -1 +0,0 @@
-requirements-2.12.txt

tests/requirements.txt

@@ -0,0 +1,11 @@
+-r ../requirements.txt
+ansible-lint==5.4.0
+apache-libcloud==3.7.0
+ara[server]==1.6.1
+dopy==0.3.7
+molecule==5.0.1
+molecule-plugins[vagrant]==23.4.1
+python-vagrant==1.0.0
+pytest-testinfra==7.0.0
+tox==4.5.1
+yamllint==1.31.0

tests/scripts/testcases_prepare.sh

@@ -1,10 +1,8 @@
 #!/bin/bash
 set -euxo pipefail
 
-: ${ANSIBLE_MAJOR_VERSION:=2.12}
-
 /usr/bin/python -m pip uninstall -y ansible ansible-base ansible-core
-/usr/bin/python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt
+/usr/bin/python -m pip install -r tests/requirements.txt
 mkdir -p /.ssh
 mkdir -p cluster-dump
 mkdir -p $HOME/.ssh