CI: rework pipeline: short/extended based on labels (#11324)

* CI: reduce VM resources requests to improve scheduling

* CI: Reduce default jobs; add labels(ci-full/extended) to run more test

* CI: use jobs dependencies instead of stages

* precommit one-job

* CI: Use Kubevirt VM to run Molecule and Vagrant jobs

.gitlab-ci.yml

@@ -1,12 +1,9 @@
 ---
 stages:
   - build
-  - unit-tests
+  - test
-  - moderator
   - deploy-part1
-  - deploy-part2
+  - deploy-extended
-  - deploy-part3
-  - deploy-special
 
 variables:
   KUBESPRAY_VERSION: v2.25.0
@@ -43,15 +40,26 @@ before_script:
 
 .job: &job
   tags:
-    - packet
+    - ffci
   image: $PIPELINE_IMAGE
   artifacts:
     when: always
     paths:
       - cluster-dump/
+  needs:
+    - pipeline-image
+
+.job-moderated:
+  extends: .job
+  needs:
+    - pipeline-image
+    - ci-not-authorized
+    - check-galaxy-version  # lint
+    - pre-commit            # lint
+    - vagrant-validate      # lint
 
 .testcases: &testcases
-  <<: *job
+  extends: .job-moderated
   retry: 1
   interruptible: true
   before_script:
@@ -61,12 +69,12 @@ before_script:
   script:
     - ./tests/scripts/testcases_run.sh
   after_script:
-    - chronic ./tests/scripts/testcases_cleanup.sh
+    - ./tests/scripts/testcases_cleanup.sh
 
 # For failfast, at least 1 job must be defined in .gitlab-ci.yml
 # Premoderated with manual actions
 ci-not-authorized:
-  stage: moderator
+  stage: build
   before_script: []
   after_script: []
   rules:
@@ -87,8 +95,8 @@ ci-not-authorized:
   script:
     - exit $CI_OK_TO_TEST
   tags:
-    - light
+    - ffci
-
+  needs: []
 
 include:
   - .gitlab-ci/build.yml

.gitlab-ci/build.yml

@@ -5,7 +5,7 @@
     paths:
       - image-cache
   tags:
-    - packet
+    - ffci
   stage: build
   image:
     name: gcr.io/kaniko-project/executor:debug

.gitlab-ci/lint.yml

@@ -1,29 +1,23 @@
 ---
-generate-pre-commit:
+pre-commit:
-  image: 'mikefarah/yq@sha256:bcb889a1f9bdb0613c8a054542d02360c2b1b35521041be3e1bd8fbd0534d411'
+  stage: test
-  stage: build
+  tags:
-  before_script: []
+  - ffci
+  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
+  variables:
+    PRE_COMMIT_HOME: /pre-commit-cache
   script:
-    - >
+  - pre-commit run --all-files
-      yq -r < .pre-commit-config.yaml '.repos[].hooks[].id' |
+  cache:
-      sed 's/^/      - /' |
+    key: pre-commit-all
-      cat .gitlab-ci/pre-commit-dynamic-stub.yml - > pre-commit-generated.yml
-  artifacts:
     paths:
-      - pre-commit-generated.yml
+    - /pre-commit-cache
-
+  needs: []
-run-pre-commit:
-  stage: unit-tests
-  trigger:
-    include:
-      - artifact: pre-commit-generated.yml
-        job: generate-pre-commit
-    strategy: depend
 
 vagrant-validate:
   extends: .job
-  stage: unit-tests
+  stage: test
-  tags: [light]
+  tags: [ffci]
   variables:
     VAGRANT_VERSION: 2.3.7
   script:
@@ -33,8 +27,9 @@ vagrant-validate:
 
 # TODO: convert to pre-commit hook
 check-galaxy-version:
-  stage: unit-tests
+  needs: []
-  tags: [light]
+  stage: test
+  tags: [ffci]
   image: python:3
   script:
   - tests/scripts/check_galaxy_version.sh

.gitlab-ci/molecule.yml

@@ -1,19 +1,30 @@
 ---
-
 .molecule:
-  tags: [c3.small.x86]
+  tags: [ffci-vm-med]
   only: [/^pr-.*$/]
   except: ['triggers']
-  image: $PIPELINE_IMAGE
+  image: quay.io/kubespray/vm-kubespray-ci:v6
   services: []
   stage: deploy-part1
+  needs: []
+  # - ci-not-authorized
+  variables:
+    VAGRANT_DEFAULT_PROVIDER: "libvirt"
   before_script:
-    - tests/scripts/rebase.sh
+  - groups
+  - python3 -m venv citest
+  - source citest/bin/activate
+  - vagrant plugin expunge --reinstall --force --no-tty
+  - vagrant plugin install vagrant-libvirt
+  - pip install --no-compile --no-cache-dir pip -U
+  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
+  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
+  - ./tests/scripts/rebase.sh
   - ./tests/scripts/vagrant_clean.sh
   script:
   - ./tests/scripts/molecule_run.sh
   after_script:
-    - chronic ./tests/scripts/molecule_logs.sh
+  - ./tests/scripts/molecule_logs.sh
   artifacts:
     when: always
     paths:
@@ -21,6 +32,7 @@
 
 # CI template for periodic CI jobs
 # Enabled when PERIODIC_CI_ENABLED var is set
+
 .molecule_periodic:
   only:
     variables:
@@ -51,24 +63,24 @@ molecule_containerd:
 
 molecule_cri-o:
   extends: .molecule
-  stage: deploy-part2
+  stage: deploy-part1
   script:
   - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
   allow_failure: true
   when: on_success
 
-# Stage 3 container engines don't get as much attention so allow them to fail
+# # Stage 3 container engines don't get as much attention so allow them to fail
-molecule_kata:
+# molecule_kata:
-  extends: .molecule
+#   extends: .molecule
-  stage: deploy-part3
+#   stage: deploy-extended
-  script:
+#   script:
-    - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
+#     - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
-  when: manual
+#   when: manual
-# FIXME: this test is broken (perma-failing)
+# # FIXME: this test is broken (perma-failing)
 
 molecule_gvisor:
   extends: .molecule
-  stage: deploy-part3
+  stage: deploy-extended
   script:
   - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
   when: manual
@@ -76,7 +88,7 @@ molecule_gvisor:
 
 molecule_youki:
   extends: .molecule
-  stage: deploy-part3
+  stage: deploy-extended
   script:
   - ./tests/scripts/molecule_run.sh -i container-engine/youki
   when: manual

.gitlab-ci/packet.yml

@@ -6,14 +6,56 @@
     CI_PLATFORM: packet
     SSH_USER: kubespray
   tags:
-    - packet
+    - ffci
-  except: [triggers]
+  needs:
+    - pipeline-image
+    - ci-not-authorized
 
 # CI template for PRs
 .packet_pr:
-  only: [/^pr-.*$/]
+  stage: deploy-part1
+  rules:
+    - if: $PR_LABELS =~ /.*ci-short.*/
+      when: manual
+      allow_failure: true
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
   extends: .packet
 
+  ## Uncomment this to have multiple stages
+  # needs:
+  #   - packet_ubuntu20-calico-all-in-one
+
+.packet_pr_short:
+  stage: deploy-part1
+  extends: .packet
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+.packet_pr_manual:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*ci-full.*/
+      when: on_success
+    # Else run as manual
+    - when: manual
+      allow_failure: true
+
+.packet_pr_extended:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
 # CI template for periodic CI jobs
 # Enabled when PERIODIC_CI_ENABLED var is set
 .packet_periodic:
@@ -34,118 +76,87 @@ packet_cleanup_old:
 # The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
 packet_ubuntu20-calico-all-in-one:
   stage: deploy-part1
-  extends: .packet_pr
+  extends: .packet_pr_short
-  when: on_success
   variables:
     RESET_CHECK: "true"
 
 # ### PR JOBS PART2
 
-packet_ubuntu20-all-in-one-docker:
+packet_ubuntu20-crio:
-  stage: deploy-part2
+  extends: .packet_pr_manual
-  extends: .packet_pr
-  when: on_success
-
-packet_ubuntu20-calico-all-in-one-hardening:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_ubuntu22-all-in-one-docker:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
 
 packet_ubuntu22-calico-all-in-one:
-  stage: deploy-part2
   extends: .packet_pr
-  when: on_success
-
-packet_ubuntu24-all-in-one-docker:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_ubuntu24-calico-all-in-one:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
 
 packet_ubuntu24-calico-etcd-datastore:
-  stage: deploy-part2
   extends: .packet_pr
-  when: on_success
-
-packet_centos7-flannel-addons-ha:
-  extends: .packet_pr
-  stage: deploy-part2
-  when: on_success
 
 packet_almalinux8-crio:
   extends: .packet_pr
-  stage: deploy-part2
-  when: on_success
-  allow_failure: true
 
-packet_ubuntu20-crio:
+
+packet_almalinux8-kube-ovn:
   extends: .packet_pr
-  stage: deploy-part2
-  when: manual
-
-packet_fedora37-crio:
-  extends: .packet_pr
-  stage: deploy-part2
-  when: manual
-
-packet_ubuntu20-flannel-ha:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-
-packet_debian10-cilium-svc-proxy:
-  stage: deploy-part2
-  extends: .packet_periodic
-  when: on_success
-
-packet_debian10-calico:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
 
 packet_debian10-docker:
-  stage: deploy-part2
   extends: .packet_pr
-  when: on_success
 
 packet_debian11-calico:
-  stage: deploy-part2
   extends: .packet_pr
-  when: on_success
 
-packet_debian11-docker:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_debian12-calico:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_debian12-docker:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
 
 packet_debian12-cilium:
-  stage: deploy-part2
+  extends: .packet_pr
-  extends: .packet_periodic
+
-  when: on_success
+packet_rockylinux8-calico:
+  extends: .packet_pr
+
+packet_rockylinux9-cilium:
+  extends: .packet_pr
+  variables:
+    RESET_CHECK: "true"
+
+packet_amazon-linux-2-all-in-one:
+  extends: .packet_pr
+
+packet_fedora38-docker-weave:
+  extends: .packet_pr
+  allow_failure: true
+
+packet_opensuse-docker-cilium:
+  extends: .packet_pr
+
+
+packet_ubuntu20-cilium-sep:
+  extends: .packet_pr
+
+## Extended
+packet_debian11-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux8-calico-remove-node:
+  extends: .packet_pr_extended
+  variables:
+    REMOVE_NODE_CHECK: "true"
+    REMOVE_NODE_NAME: "instance-3"
+
+packet_rockylinux9-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux8-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux8-docker:
+  extends: .packet_pr_extended
 
 packet_centos7-calico-ha-once-localhost:
-  stage: deploy-part2
+  extends: .packet_pr_extended
-  extends: .packet_pr
-  when: on_success
   variables:
     # This will instruct Docker not to start over TLS.
     DOCKER_TLS_CERTDIR: ""
@@ -153,196 +164,127 @@ packet_centos7-calico-ha-once-localhost:
   services:
     - docker:26.1.4-dind
 
-packet_almalinux8-kube-ovn:
+packet_centos7-flannel-addons-ha:
-  stage: deploy-part2
+  extends: .packet_pr_extended
-  extends: .packet_pr
-  when: on_success
 
-packet_almalinux8-calico:
+packet_debian10-calico:
-  stage: deploy-part2
+  extends: .packet_pr_extended
-  extends: .packet_pr
-  when: on_success
 
-packet_rockylinux8-calico:
+packet_ubuntu20-calico-all-in-one-hardening:
-  stage: deploy-part2
+  extends: .packet_pr_extended
-  extends: .packet_pr
-  when: on_success
 
-packet_rockylinux9-calico:
+packet_ubuntu24-calico-all-in-one:
-  stage: deploy-part2
+  extends: .packet_pr_extended
-  extends: .packet_pr
-  when: on_success
 
-packet_rockylinux9-cilium:
+packet_ubuntu20-calico-etcd-kubeadm:
-  stage: deploy-part2
+  extends: .packet_pr_extended
-  extends: .packet_pr
-  when: on_success
-  variables:
-    RESET_CHECK: "true"
 
-packet_almalinux8-docker:
+packet_ubuntu24-all-in-one-docker:
-  stage: deploy-part2
+  extends: .packet_pr_extended
-  extends: .packet_pr
-  when: on_success
 
-packet_amazon-linux-2-all-in-one:
+packet_ubuntu22-all-in-one-docker:
-  stage: deploy-part2
+  extends: .packet_pr_extended
-  extends: .packet_pr
-  when: on_success
-
-packet_fedora38-docker-weave:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-  allow_failure: true
-
-packet_opensuse-docker-cilium:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
 
 # ### MANUAL JOBS
+packet_centos7-calico-ha:
+  extends: .packet_pr_manual
+
+packet_fedora37-crio:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-flannel-ha:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-all-in-one-docker:
+  extends: .packet_pr_manual
 
 packet_ubuntu20-docker-weave-sep:
-  stage: deploy-part2
+  extends: .packet_pr_manual
-  extends: .packet_pr
-  when: manual
-
-packet_ubuntu20-cilium-sep:
-  stage: deploy-special
-  extends: .packet_pr
-  when: manual
 
 packet_ubuntu20-flannel-ha-once:
-  stage: deploy-part2
+  extends: .packet_pr_manual
-  extends: .packet_pr
+
-  when: manual
+packet_fedora37-calico-swap-selinux:
+  extends: .packet_pr_manual
 
-# Calico HA eBPF
 packet_almalinux8-calico-ha-ebpf:
-  stage: deploy-part2
+  extends: .packet_pr_manual
-  extends: .packet_pr
+
-  when: manual
+packet_almalinux8-calico-nodelocaldns-secondary:
+  extends: .packet_pr_manual
 
 packet_debian10-macvlan:
-  stage: deploy-part2
+  extends: .packet_pr_manual
-  extends: .packet_pr
-  when: manual
-
-packet_centos7-calico-ha:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
 
 packet_centos7-multus-calico:
-  stage: deploy-part2
+  extends: .packet_pr_manual
-  extends: .packet_pr
-  when: manual
 
+packet_debian11-custom-cni:
+  extends: .packet_pr_manual
+
+packet_debian11-kubelet-csr-approver:
+  extends: .packet_pr_manual
+
+packet_debian12-custom-cni-helm:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-calico-ha-wireguard:
+  extends: .packet_pr_manual
+
+# PERIODIC
 packet_fedora38-docker-calico:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .packet_periodic
-  when: on_success
   variables:
     RESET_CHECK: "true"
 
 packet_fedora37-calico-selinux:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .packet_periodic
-  when: on_success
-
-packet_fedora37-calico-swap-selinux:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-
-packet_almalinux8-calico-nodelocaldns-secondary:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
 
 packet_fedora38-kube-ovn:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .packet_periodic
-  when: on_success
-
-packet_debian11-custom-cni:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-
-packet_debian11-kubelet-csr-approver:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-
-packet_debian12-custom-cni-helm:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-
-# ### PR JOBS PART3
-# Long jobs (45min+)
 
 packet_centos7-weave-upgrade-ha:
-  stage: deploy-part3
+  stage: deploy-extended
   extends: .packet_periodic
-  when: on_success
   variables:
     UPGRADE_TEST: basic
 
 packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
-  stage: deploy-part3
+  stage: deploy-extended
   extends: .packet_periodic
-  when: on_success
   variables:
     UPGRADE_TEST: basic
 
-# Calico HA Wireguard
-packet_ubuntu20-calico-ha-wireguard:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-
-packet_debian11-calico-upgrade:
-  stage: deploy-part3
-  extends: .packet_pr
-  when: on_success
-  variables:
-    UPGRADE_TEST: graceful
-
-packet_almalinux8-calico-remove-node:
-  stage: deploy-part3
-  extends: .packet_pr
-  when: on_success
-  variables:
-    REMOVE_NODE_CHECK: "true"
-    REMOVE_NODE_NAME: "instance-3"
-
-packet_ubuntu20-calico-etcd-kubeadm:
-  stage: deploy-part3
-  extends: .packet_pr
-  when: on_success
 
 packet_debian11-calico-upgrade-once:
-  stage: deploy-part3
+  stage: deploy-extended
   extends: .packet_periodic
-  when: on_success
   variables:
     UPGRADE_TEST: graceful
 
 packet_ubuntu20-calico-ha-recover:
-  stage: deploy-part3
+  stage: deploy-extended
   extends: .packet_periodic
-  when: on_success
   variables:
     RECOVER_CONTROL_PLANE_TEST: "true"
     RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
 
 packet_ubuntu20-calico-ha-recover-noquorum:
-  stage: deploy-part3
+  stage: deploy-extended
   extends: .packet_periodic
-  when: on_success
   variables:
     RECOVER_CONTROL_PLANE_TEST: "true"
     RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
+
+packet_debian10-cilium-svc-proxy:
+  stage: deploy-extended
+  extends: .packet_periodic
+
+packet_debian11-calico-upgrade:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: graceful

.gitlab-ci/pre-commit-dynamic-stub.yml

@@ -7,7 +7,7 @@ pre-commit:
   variables:
     PRE_COMMIT_HOME: /pre-commit-cache
   script:
-  - pre-commit run -a $HOOK_ID
+  - pre-commit run --all-files
   cache:
     key: pre-commit-$HOOK_ID
     paths:

.gitlab-ci/terraform.yml

@@ -2,6 +2,10 @@
 # Tests for contrib/terraform/
 .terraform_install:
   extends: .job
+  needs:
+    - ci-not-authorized
+    - pipeline-image
+  stage: deploy-part1
   before_script:
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh
@@ -24,17 +28,19 @@
 
 .terraform_validate:
   extends: .terraform_install
-  stage: unit-tests
+  tags: [ffci]
-  tags: [light]
   only: ['master', /^pr-.*$/]
   script:
     - terraform -chdir="contrib/terraform/$PROVIDER" validate
     - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+  stage: test
+  needs:
+    - pipeline-image
 
 .terraform_apply:
   extends: .terraform_install
-  tags: [light]
+  tags: [ffci]
-  stage: deploy-part3
+  stage: deploy-extended
   when: manual
   only: [/^pr-.*$/]
   artifacts:
@@ -51,7 +57,7 @@
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - chronic ./tests/scripts/testcases_cleanup.sh
+    - ./tests/scripts/testcases_cleanup.sh
 
 tf-validate-openstack:
   extends: .terraform_validate
@@ -146,8 +152,7 @@ tf-validate-nifcloud:
   TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df"
 
 tf-elastx_cleanup:
-  stage: unit-tests
+  tags: [ffci]
-  tags: [light]
   image: python
   variables:
     <<: *elastx_variables
@@ -155,10 +160,11 @@ tf-elastx_cleanup:
     - pip install -r scripts/openstack-cleanup/requirements.txt
   script:
     - ./scripts/openstack-cleanup/main.py
+  allow_failure: true
 
 tf-elastx_ubuntu20-calico:
   extends: .terraform_apply
-  stage: deploy-part3
+  stage: deploy-part1
   when: on_success
   allow_failure: true
   variables:

.gitlab-ci/vagrant.yml

@@ -1,64 +1,73 @@
 ---
-
 .vagrant:
   extends: .testcases
+  needs:
+    - ci-not-authorized
   variables:
     CI_PLATFORM: "vagrant"
     SSH_USER: "vagrant"
     VAGRANT_DEFAULT_PROVIDER: "libvirt"
     KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
-  tags: [c3.small.x86]
+    DOCKER_NAME: vagrant
-  only: [/^pr-.*$/]
+    VAGRANT_ANSIBLE_TAGS: facts
-  except: ['triggers']
+  tags: [ffci-vm-large]
-  image: $PIPELINE_IMAGE
+  # only: [/^pr-.*$/]
+  # except: ['triggers']
+  image: quay.io/kubespray/vm-kubespray-ci:v6
   services: []
   before_script:
+    - echo $USER
+    - python3 -m venv citest
+    - source citest/bin/activate
+    - vagrant plugin expunge --reinstall --force --no-tty
+    - vagrant plugin install vagrant-libvirt
+    - pip install --no-compile --no-cache-dir pip -U
+    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
+    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
     - ./tests/scripts/testcases_run.sh
-  after_script:
-    - chronic ./tests/scripts/testcases_cleanup.sh
 
 vagrant_ubuntu20-calico-dual-stack:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .vagrant
   when: manual
 # FIXME: this test if broken (perma-failing)
 
 vagrant_ubuntu20-weave-medium:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .vagrant
   when: manual
 
 vagrant_ubuntu20-flannel:
-  stage: deploy-part2
+  stage: deploy-part1
   extends: .vagrant
   when: on_success
   allow_failure: false
 
 vagrant_ubuntu20-flannel-collection:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .vagrant
-  when: on_success
+  when: manual
 
 vagrant_ubuntu20-kube-router-sep:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .vagrant
   when: manual
 
 # Service proxy test fails connectivity testing
 vagrant_ubuntu20-kube-router-svc-proxy:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .vagrant
   when: manual
 
 vagrant_fedora37-kube-router:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .vagrant
   when: manual
 # FIXME: this test if broken (perma-failing)
 
 vagrant_centos7-kube-router:
-  stage: deploy-part2
+  stage: deploy-extended
   extends: .vagrant
   when: manual

Vagrantfile

@@ -278,6 +278,7 @@ Vagrant.configure("2") do |config|
         "local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}",
         "local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}",
         "ansible_ssh_user": SUPPORTED_OS[$os][:user],
+        "ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"),
         "unsafe_show_logs": "True"
       }
 

docs/developers/ci-setup.md

@@ -5,8 +5,8 @@
 1. build: build a docker image to be used in the pipeline
 2. unit-tests: fast jobs for fast feedback (linting, etc...)
 3. deploy-part1: small number of jobs to test if the PR works with default settings
-4. deploy-part2: slow jobs testing different platforms, OS, settings, CNI, etc...
+4. deploy-extended: slow jobs testing different platforms, OS, settings, CNI, etc...
-5. deploy-part3: very slow jobs (upgrades, etc...)
+5. deploy-extended: very slow jobs (upgrades, etc...)
 
 ## Runners
 

pipeline.Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:jammy-20230308
 # Pip needs this as well at the moment to install ansible
 # (and potentially other packages)
 # See: https://github.com/pypa/pip/issues/10219
-ENV VAGRANT_VERSION=2.3.7 \
+ENV VAGRANT_VERSION=2.4.1 \
     VAGRANT_DEFAULT_PROVIDER=libvirt \
     VAGRANT_ANSIBLE_TAGS=facts \
     LANG=C.UTF-8 \
@@ -31,6 +31,8 @@ RUN apt update -q \
          unzip \
          libvirt-clients \
          qemu-utils \
+         qemu-kvm \
+         dnsmasq \
     && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
     && apt update -q \
@@ -46,6 +48,7 @@ ADD ./roles/kubespray-defaults/defaults/main/main.yml /kubespray/roles/kubespray
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
     && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
+    && pip install --no-compile --no-cache-dir -r requirements.txt \
     && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \
     && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
     && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \

tests/Makefile

@@ -35,8 +35,7 @@ cleanup-packet:
 
 create-vagrant:
 	vagrant up
-	find / -name vagrant_ansible_inventory
+	cp  $(CI_PROJECT_DIR)/inventory/sample/vagrant_ansible_inventory $(INVENTORY)
-	cp  /builds/kargo-ci/kubernetes-sigs-kubespray/inventory/sample/vagrant_ansible_inventory $(INVENTORY)
 
 delete-vagrant:
 	vagrant destroy -f

tests/cloud_playbooks/roles/packet-ci/defaults/main.yml

@@ -4,14 +4,13 @@
 vm_cpu_cores: 2
 vm_cpu_sockets: 1
 vm_cpu_threads: 2
-vm_memory: 2048Mi
+vm_memory: 2048
 
 # Replace invalid characters so that we can use the branch name in kubernetes labels
 branch_name_sane: "{{ branch | regex_replace('/', '-') }}"
 
 # Request/Limit allocation settings
-
+cpu_allocation_ratio: 0.25
-cpu_allocation_ratio: 0.5
 memory_allocation_ratio: 1
 
 # Default path for inventory

tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2

@@ -4,6 +4,8 @@ kind: VirtualMachine
 metadata:
   name: "instance-{{ vm_id }}"
   namespace: "{{ test_name }}"
+  annotations:
+    kubespray.com/ci.template-path: "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2"
   labels:
     kubevirt.io/os: {{ cloud_image }}
 spec:
@@ -34,10 +36,10 @@ spec:
             threads: {{ vm_cpu_threads }}
         resources:
           requests:
-            memory: {{ vm_memory * memory_allocation_ratio }}
+            memory: "{{ vm_memory * memory_allocation_ratio }}Mi"
             cpu: {{ vm_cpu_cores * cpu_allocation_ratio }}
           limits:
-            memory: {{ vm_memory }}
+            memory: "{{ vm_memory }}Mi"
             cpu: {{ vm_cpu_cores }}
       networks:
       - name: default

tests/files/packet_almalinux8-calico-ha-ebpf.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: ha
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 calico_bpf_enabled: true

tests/files/packet_almalinux8-calico-nodelocaldns-secondary.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 enable_nodelocaldns_secondary: true

tests/files/packet_almalinux8-calico.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 metrics_server_enabled: true

tests/files/packet_almalinux8-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Use docker
 container_manager: docker

tests/files/packet_almalinux8-kube-ovn.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 kube_network_plugin: kube-ovn

tests/files/packet_rockylinux8-calico.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: rockylinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 metrics_server_enabled: true

tests/files/packet_rockylinux9-calico.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: rockylinux-9
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 metrics_server_enabled: true

tests/files/packet_rockylinux9-cilium.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: rockylinux-9
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 kube_network_plugin: cilium

tests/files/packet_ubuntu22-all-in-one-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2204
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu22-calico-all-in-one.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2204
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu24-all-in-one-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu24-calico-all-in-one.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu24-calico-etcd-datastore.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: node-etcd-client
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/vagrant_ubuntu20-flannel.rb

@@ -2,7 +2,6 @@ $os = "ubuntu2004"
 
 # For CI we are not worries about data persistence across reboot
 $libvirt_volume_cache = "unsafe"
-
 # Checking for box update can trigger API rate limiting
 # https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html
 $box_check_update = false

tests/files/vagrant_ubuntu20-flannel.yml

@@ -1,3 +1,4 @@
 ---
 # Kubespray settings
 kube_network_plugin: flannel
+ansible_ssh_private_key: .vagrant.d/insecure_private_key

tests/scripts/vagrant-install.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+# install_vagrant() {
+#  sudo apt install vagrant-libvirt vagrant -y
+#  sudo vagrant plugin install vagrant-libvirt
+# }
+
+# prep(){
+# 	sudo apt-get update -y
+# 	sudo apt-get install ca-certificates curl libvirt-daemon-system\
+# 		libvirt-clients qemu-utils qemu-kvm htop atop -y
+
+# 	sudo install -m 0755 -d /etc/apt/keyrings
+# }
+# install_docker() {
+# 	VERSION_STRING=5:26.1.0-1~ubuntu.24.04~noble
+# 	sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+# 	sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# 	# Add the repository to Apt sources:
+# 	echo \
+# 		"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+# 		$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+# 			sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+# 	sudo apt-get update -y
+
+# 	sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+# }
+# install_docker_auto () {
+# 	 curl -fsSL https://get.docker.com -o get-docker.sh
+# 	 sudo sh ./get-docker.sh --dry-run
+# }
+
+
+
+VAGRANT_VERSION=2.4.1
+VAGRANT_DEFAULT_PROVIDER=libvirt
+VAGRANT_ANSIBLE_TAGS=facts
+LANG=C.UTF-8
+DEBIAN_FRONTEND=noninteractive
+PYTHONDONTWRITEBYTECODE=1
+KUBE_VERSION=1.29.5
+pipeline_install() {
+    cp /etc/apt/sources.list /etc/apt/sources.list."$(date +"%F")"
+    sed -i -e '/^# deb-src.*universe$/s/# //g' /etc/apt/sources.list
+    sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
+
+    apt update
+    # libssl-dev \
+        # python3-dev \
+        #         # jq \
+        # moreutils \
+        # libvirt-dev \
+        #         # rsync \
+        # git \
+        #                 # htop \
+        # gpg \
+        # atop
+
+        # gnupg2 \
+# software-properties-common
+#
+    apt install --no-install-recommends -y \
+        git \
+        make \
+        python3-pip \
+        sshpass \
+        apt-transport-https \
+        openssh-client \
+        ca-certificates \
+        curl \
+        libfuse2 \
+        unzip \
+        qemu-utils \
+        libvirt-daemon-system \
+        libvirt-clients \
+        qemu-kvm \
+        ebtables libguestfs-tools \
+        ruby-fog-libvirt \
+        libvirt-dev \
+        gcc \
+        build-essential \
+        ruby-libvirt \
+        libxslt-dev libxml2-dev zlib1g-dev \
+        python3-venv python3-full \
+        dnsmasq
+
+    apt-get build-dep -y ruby-libvirt ruby-dev
+    ### VAGRANT ###
+    # apt-get install -y unzip
+    curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_linux_amd64.zip
+    unzip vagrant_${VAGRANT_VERSION}_linux_amd64.zip
+    mv vagrant /usr/local/bin/vagrant
+    chmod a+x /usr/local/bin/vagrant
+    # ls -la /usr/local/bin/vagrant
+    /usr/local/bin/vagrant plugin install vagrant-libvirt
+    usermod -aG kvm kubespray
+    usermod -aG libvirt kubespray
+
+    ### DOCKER ###
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+    add-apt-repository -y "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    apt update
+    apt install --no-install-recommends -y docker-ce
+    apt autoremove -y --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*
+
+    ### KUBECTL ###
+    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+    mv kubectl /usr/local/bin/kubectl
+    chmod a+x /usr/local/bin/kubectl
+    systemctl restart libvirtd
+    # Install Vagrant
+     # apt update -y
+    # echo apt-get install -y unzip libfuse2 vagrant vagrant-libvirt
+    # apt --fix-broken install -y
+    # dpkg --configure -a -y
+
+
+}
+# wrapped up in a function so that we have some protection against only getting
+# half the file during "curl | sh"
+pipeline_install