[CI] Add a CI job to test cluster upgrading, and fix bug of testcases_run.sh (#11458)

* Fix: fix testcases_run.sh for upgrade tests

Need to git checkout ${CI_COMMIT_SHA} before running upgrade playbook (revert #11173 partially)

* feat: add CI job to test upgrade

Add a packet_ubuntu22-calico-all-in-one-upgrade job

.gitlab-ci/packet.yml

@@ -88,6 +88,11 @@ packet_ubuntu20-crio:
 packet_ubuntu22-calico-all-in-one:
   extends: .packet_pr
 
+packet_ubuntu22-calico-all-in-one-upgrade:
+  extends: .packet_pr
+  variables:
+    UPGRADE_TEST: graceful
+
 packet_ubuntu24-calico-etcd-datastore:
   extends: .packet_pr
 

tests/files/packet_ubuntu22-calico-all-in-one-upgrade.yml

@@ -0,0 +1,24 @@
+---
+# Instance settings
+cloud_image: ubuntu-2204
+mode: all-in-one
+vm_memory: 1600
+
+# Kubespray settings
+auto_renew_certificates: true
+
+# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko
+kube_proxy_mode: iptables
+enable_nodelocaldns: false
+
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://mirror.gcr.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+  - prefix: 172.19.16.11:5000
+    mirrors:
+      - host: http://172.19.16.11:5000
+        capabilities: ["pull", "resolve", "push"]
+        skip_verify: true

tests/scripts/testcases_run.sh

@@ -42,12 +42,13 @@ if [[ "$CI_JOB_NAME" =~ "opensuse" ]]; then
 fi
 
 # Check out latest tag if testing upgrade
-test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout "$KUBESPRAY_VERSION"
+if [ "${UPGRADE_TEST}" != "false" ]; then
+  git fetch --all && git checkout "$KUBESPRAY_VERSION"
   # Checkout the CI vars file so it is available
-test "${UPGRADE_TEST}" != "false" && git checkout "${CI_COMMIT_SHA}" tests/files/${CI_JOB_NAME}.yml
-test "${UPGRADE_TEST}" != "false" && git checkout "${CI_COMMIT_SHA}" ${CI_TEST_REGISTRY_MIRROR}
-test "${UPGRADE_TEST}" != "false" && git checkout "${CI_COMMIT_SHA}" ${CI_TEST_SETTING}
-
+  git checkout "${CI_COMMIT_SHA}" tests/files/${CI_JOB_NAME}.yml
+  git checkout "${CI_COMMIT_SHA}" ${CI_TEST_REGISTRY_MIRROR}
+  git checkout "${CI_COMMIT_SHA}" ${CI_TEST_SETTING}
+fi
 
 run_playbook () {
 playbook=$1
@@ -67,8 +68,10 @@ ansible-playbook --limit "all:!fake_hosts" \
 run_playbook cluster.yml
 
 # Repeat deployment if testing upgrade
-case "${UPGRADE_TEST}" in
+if [ "${UPGRADE_TEST}" != "false" ]; then
+  git checkout "${CI_COMMIT_SHA}"
 
+  case "${UPGRADE_TEST}" in
     "basic")
         run_playbook cluster.yml
         ;;
@@ -78,6 +81,7 @@ case "${UPGRADE_TEST}" in
     *)
         ;;
   esac
+fi
 
 # Test control plane recovery
 if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then