From 0d0468e127733252895473c564fa6cd4c348d102 Mon Sep 17 00:00:00 2001
From: Cristian Calin <6627509+cristicalin@users.noreply.github.com>
Date: Thu, 11 Nov 2021 02:11:50 +0200
Subject: [PATCH] Exercise multiple ansible versions in CI (#8172)

* Ansible: separate requirements files for supported ansible versions

* Ansible: allow using ansible 2.11

* CI: Exercise Ansible 2.9 and Ansible 2.11 in a basic AIO CI job

* CI: Allow running a reset test outside of idempotency tests and running it in stage1

* CI: move ubuntu18-calico-aio job to stage2 and relay only on ubuntu20 with the variously supported ansible versions for stage1

* CI: add capability to install collections or roles from ansible-galaxy to mitigate missing behavior in older ansible versions
---
 .gitlab-ci.yml                                |  5 +--
 .gitlab-ci/packet.yml                         | 23 ++++++++++++--
 ansible_version.yml                           |  2 +-
 requirements-2.10.txt                         | 10 ++++++
 requirements-2.11.txt                         | 10 ++++++
 requirements-2.9.txt                          |  8 +++++
 requirements-2.9.yml                          |  4 +++
 requirements.txt                              | 11 +------
 ...acket_ubuntu20-calico-aio-ansible-2_11.yml |  1 +
 ...packet_ubuntu20-calico-aio-ansible-2_9.yml |  1 +
 tests/requirements-2.10.txt                   | 12 +++++++
 tests/requirements-2.11.txt                   | 12 +++++++
 tests/requirements-2.9.txt                    | 12 +++++++
 tests/requirements.txt                        | 13 +-------
 tests/scripts/testcases_prepare.sh            | 13 ++++++--
 tests/scripts/testcases_run.sh                | 31 +++++++++----------
 16 files changed, 123 insertions(+), 45 deletions(-)
 create mode 100644 requirements-2.10.txt
 create mode 100644 requirements-2.11.txt
 create mode 100644 requirements-2.9.txt
 create mode 100644 requirements-2.9.yml
 mode change 100644 => 120000 requirements.txt
 create mode 120000 tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml
 create mode 120000 tests/files/packet_ubuntu20-calico-aio-ansible-2_9.yml
 create mode 100644 tests/requirements-2.10.txt
 create mode 100644 tests/requirements-2.11.txt
 create mode 100644 tests/requirements-2.9.txt
 mode change 100644 => 120000 tests/requirements.txt

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3a2b56628..ae5a65bec 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,12 +32,13 @@ variables:
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
   TERRAFORM_VERSION: 1.0.8
+  ANSIBLE_MAJOR_VERSION: "2.10"
 
 before_script:
   - ./tests/scripts/rebase.sh
   - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-  - python -m pip uninstall -y ansible
-  - python -m pip install -r tests/requirements.txt
+  - python -m pip uninstall -y ansible ansible-base ansible-core
+  - python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt
   - mkdir -p /.ssh
 
 .job: &job
diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml
index 6e72a4cd8..90a3ad397 100644
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@@ -23,15 +23,34 @@
   extends: .packet
 
 packet_ubuntu18-calico-aio:
-  stage: deploy-part1
+  stage: deploy-part2
   extends: .packet_pr
   when: on_success
 
-# Future AIO job
+# The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken
 packet_ubuntu20-calico-aio:
   stage: deploy-part1
   extends: .packet_pr
   when: on_success
+  variables:
+    RESET_CHECK: "true"
+
+# Exericse ansible variants
+packet_ubuntu20-calico-aio-ansible-2_9:
+  stage: deploy-part1
+  extends: .packet_pr
+  when: on_success
+  variables:
+    ANSIBLE_MAJOR_VERSION: "2.9"
+    RESET_CHECK: "true"
+
+packet_ubuntu20-calico-aio-ansible-2_11:
+  stage: deploy-part1
+  extends: .packet_pr
+  when: on_success
+  variables:
+    ANSIBLE_MAJOR_VERSION: "2.11"
+    RESET_CHECK: "true"
 
 # ### PR JOBS PART2
 
diff --git a/ansible_version.yml b/ansible_version.yml
index b7fff0039..5226fd90f 100644
--- a/ansible_version.yml
+++ b/ansible_version.yml
@@ -5,7 +5,7 @@
   vars:
     minimal_ansible_version: 2.9.0
     minimal_ansible_version_2_10: 2.10.11
-    maximal_ansible_version: 2.11.0
+    maximal_ansible_version: 2.12.0
     ansible_connection: local
   tags: always
   tasks:
diff --git a/requirements-2.10.txt b/requirements-2.10.txt
new file mode 100644
index 000000000..5fcbf8048
--- /dev/null
+++ b/requirements-2.10.txt
@@ -0,0 +1,10 @@
+ansible==3.4.0
+ansible-base==2.10.15
+cryptography==2.8
+jinja2==2.11.3
+netaddr==0.7.19
+pbr==5.4.4
+jmespath==0.9.5
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.4
+MarkupSafe==1.1.1
diff --git a/requirements-2.11.txt b/requirements-2.11.txt
new file mode 100644
index 000000000..5d535be65
--- /dev/null
+++ b/requirements-2.11.txt
@@ -0,0 +1,10 @@
+ansible==4.8.0
+ansible-core==2.11.6
+cryptography==2.8
+jinja2==2.11.3
+netaddr==0.7.19
+pbr==5.4.4
+jmespath==0.9.5
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.4
+MarkupSafe==1.1.1
diff --git a/requirements-2.9.txt b/requirements-2.9.txt
new file mode 100644
index 000000000..220780881
--- /dev/null
+++ b/requirements-2.9.txt
@@ -0,0 +1,8 @@
+ansible==2.9.27
+jinja2==2.11.3
+netaddr==0.7.19
+pbr==5.4.4
+jmespath==0.9.5
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.4
+MarkupSafe==1.1.1
diff --git a/requirements-2.9.yml b/requirements-2.9.yml
new file mode 100644
index 000000000..e8034543d
--- /dev/null
+++ b/requirements-2.9.yml
@@ -0,0 +1,4 @@
+---
+collections:
+- name: community.general
+  version: '<3.0'
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index ab7f293f1..000000000
--- a/requirements.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-ansible==3.4.0
-ansible-base==2.10.11
-cryptography==2.8
-jinja2==2.11.3
-netaddr==0.7.19
-pbr==5.4.4
-jmespath==0.9.5
-ruamel.yaml==0.16.10
-ruamel.yaml.clib==0.2.4
-MarkupSafe==1.1.1
diff --git a/requirements.txt b/requirements.txt
new file mode 120000
index 000000000..5202ea4fb
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+requirements-2.10.txt
\ No newline at end of file
diff --git a/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml b/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml
new file mode 120000
index 000000000..10064637f
--- /dev/null
+++ b/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml
@@ -0,0 +1 @@
+packet_ubuntu20-calico-aio.yml
\ No newline at end of file
diff --git a/tests/files/packet_ubuntu20-calico-aio-ansible-2_9.yml b/tests/files/packet_ubuntu20-calico-aio-ansible-2_9.yml
new file mode 120000
index 000000000..10064637f
--- /dev/null
+++ b/tests/files/packet_ubuntu20-calico-aio-ansible-2_9.yml
@@ -0,0 +1 @@
+packet_ubuntu20-calico-aio.yml
\ No newline at end of file
diff --git a/tests/requirements-2.10.txt b/tests/requirements-2.10.txt
new file mode 100644
index 000000000..a80c3a868
--- /dev/null
+++ b/tests/requirements-2.10.txt
@@ -0,0 +1,12 @@
+-r ../requirements-2.10.txt
+yamllint==1.19.0
+apache-libcloud==2.2.1
+tox==3.11.1
+dopy==0.3.7
+cryptography==2.8
+ansible-lint==5.0.11
+openshift==0.8.8
+molecule==3.0.6
+molecule-vagrant==0.3
+testinfra==5.2.2
+python-vagrant==0.5.15
diff --git a/tests/requirements-2.11.txt b/tests/requirements-2.11.txt
new file mode 100644
index 000000000..8ad98b8b4
--- /dev/null
+++ b/tests/requirements-2.11.txt
@@ -0,0 +1,12 @@
+-r ../requirements-2.11.txt
+yamllint==1.19.0
+apache-libcloud==2.2.1
+tox==3.11.1
+dopy==0.3.7
+cryptography==2.8
+ansible-lint==5.0.11
+openshift==0.8.8
+molecule==3.0.6
+molecule-vagrant==0.3
+testinfra==5.2.2
+python-vagrant==0.5.15
diff --git a/tests/requirements-2.9.txt b/tests/requirements-2.9.txt
new file mode 100644
index 000000000..291307249
--- /dev/null
+++ b/tests/requirements-2.9.txt
@@ -0,0 +1,12 @@
+-r ../requirements-2.9.txt
+yamllint==1.19.0
+apache-libcloud==2.2.1
+tox==3.11.1
+dopy==0.3.7
+cryptography==2.8
+ansible-lint==5.0.11
+openshift==0.8.8
+molecule==3.0.6
+molecule-vagrant==0.3
+testinfra==5.2.2
+python-vagrant==0.5.15
diff --git a/tests/requirements.txt b/tests/requirements.txt
deleted file mode 100644
index 2524ef93c..000000000
--- a/tests/requirements.txt
+++ /dev/null
@@ -1,12 +0,0 @@
--r ../requirements.txt
-yamllint==1.19.0
-apache-libcloud==2.2.1
-tox==3.11.1
-dopy==0.3.7
-cryptography==2.8
-ansible-lint==5.0.11
-openshift==0.8.8
-molecule==3.0.6
-molecule-vagrant==0.3
-testinfra==5.2.2
-python-vagrant==0.5.15
diff --git a/tests/requirements.txt b/tests/requirements.txt
new file mode 120000
index 000000000..5202ea4fb
--- /dev/null
+++ b/tests/requirements.txt
@@ -0,0 +1 @@
+requirements-2.10.txt
\ No newline at end of file
diff --git a/tests/scripts/testcases_prepare.sh b/tests/scripts/testcases_prepare.sh
index d70086a2b..bfaf65f18 100755
--- a/tests/scripts/testcases_prepare.sh
+++ b/tests/scripts/testcases_prepare.sh
@@ -1,9 +1,18 @@
 #!/bin/bash
 set -euxo pipefail
 
-/usr/bin/python -m pip uninstall -y ansible
-/usr/bin/python -m pip install -r tests/requirements.txt
+: ${ANSIBLE_MAJOR_VERSION:=2.10}
+
+/usr/bin/python -m pip uninstall -y ansible ansible-base ansible-core
+/usr/bin/python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt
 mkdir -p /.ssh
 mkdir -p cluster-dump
 mkdir -p $HOME/.ssh
 ansible-playbook --version
+
+# in some cases we may need to bring in collections or roles from ansible-galaxy
+# to compensate for missing functionality in older ansible versions
+if [ -f requirements-${ANSIBLE_MAJOR_VERSION}.yml ] ; then
+  ansible-galaxy role install -r requirements-${ANSIBLE_MAJOR_VERSION}.yml
+  ansible-galaxy collection install -r requirements-${ANSIBLE_MAJOR_VERSION}.yml
+fi
diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh
index 2461d29c6..a6b595edb 100755
--- a/tests/scripts/testcases_run.sh
+++ b/tests/scripts/testcases_run.sh
@@ -86,27 +86,26 @@ ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} ${CI_TEST_ADDITIO
 ## Kubernetes conformance tests
 ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/100_check-k8s-conformance.yml $ANSIBLE_LOG_LEVEL
 
-## Idempotency checks 1/5 (repeat deployment)
 if [ "${IDEMPOT_CHECK}" = "true" ]; then
+  ## Idempotency checks 1/5 (repeat deployment)
   ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} ${CI_TEST_ADDITIONAL_VARS} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
-fi
 
-## Idempotency checks 2/5 (Advanced DNS checks)
-if [ "${IDEMPOT_CHECK}" = "true" ]; then
+  ## Idempotency checks 2/5 (Advanced DNS checks)
   ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml
+
+  if [ "${RESET_CHECK}" = "true" ]; then
+    ## Idempotency checks 3/5 (reset deployment)
+    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR}  -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
+
+    ## Idempotency checks 4/5 (redeploy after reset)
+    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+
+    ## Idempotency checks 5/5 (Advanced DNS checks)
+    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml
+  fi
 fi
 
-## Idempotency checks 3/5 (reset deployment)
-if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
+# Clean up at the end, this is to allow stage1 tests to include cleanup test
+if [ "${RESET_CHECK}" = "true" ]; then
   ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR}  -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
 fi
-
-## Idempotency checks 4/5 (redeploy after reset)
-if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
-fi
-
-## Idempotency checks 5/5 (Advanced DNS checks)
-if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml
-fi