From 72a926a38bfe73a0dcbe23ac4f68eaf8f93d7330 Mon Sep 17 00:00:00 2001 From: ant31 <2t.antoine@gmail.com> Date: Tue, 11 Jun 2024 16:02:45 +0200 Subject: [PATCH] CI: Use Kubevirt VM to run Molecule and Vagrant jobs Cherry-picked-by: Max Gautier --- .gitlab-ci.yml | 2 +- .gitlab-ci/molecule.yml | 42 +++++--- .gitlab-ci/terraform.yml | 2 +- .gitlab-ci/vagrant.yml | 22 ++-- Vagrantfile | 4 +- pipeline.Dockerfile | 5 +- tests/Makefile | 3 +- tests/files/vagrant_ubuntu20-flannel.rb | 1 - tests/files/vagrant_ubuntu20-flannel.yml | 1 + tests/scripts/vagrant-install.sh | 122 +++++++++++++++++++++++ 10 files changed, 173 insertions(+), 31 deletions(-) create mode 100755 tests/scripts/vagrant-install.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e5d17bc92..826f935af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,7 +61,7 @@ before_script: script: - ./tests/scripts/testcases_run.sh after_script: - - chronic ./tests/scripts/testcases_cleanup.sh + - ./tests/scripts/testcases_cleanup.sh # For failfast, at least 1 job must be defined in .gitlab-ci.yml # Premoderated with manual actions diff --git a/.gitlab-ci/molecule.yml b/.gitlab-ci/molecule.yml index 3961be11a..2fa94acd6 100644 --- a/.gitlab-ci/molecule.yml +++ b/.gitlab-ci/molecule.yml @@ -1,30 +1,40 @@ --- - .molecule: - tags: [c3.small.x86] + tags: [ffci-vm-med] only: [/^pr-.*$/] except: ['triggers'] - image: $PIPELINE_IMAGE + image: quay.io/kubespray/vm-kubespray-ci:v6 services: [] stage: deploy-part1 + variables: + VAGRANT_DEFAULT_PROVIDER: "libvirt" before_script: - - tests/scripts/rebase.sh - - ./tests/scripts/vagrant_clean.sh + - groups + - python3 -m venv citest + - source citest/bin/activate + - vagrant plugin expunge --reinstall --force --no-tty + - vagrant plugin install vagrant-libvirt + - pip install --no-compile --no-cache-dir pip -U + - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt + - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt + - ./tests/scripts/rebase.sh + - ./tests/scripts/vagrant_clean.sh script: - - ./tests/scripts/molecule_run.sh + - ./tests/scripts/molecule_run.sh after_script: - - chronic ./tests/scripts/molecule_logs.sh + - ./tests/scripts/molecule_logs.sh artifacts: when: always paths: - - molecule_logs/ + - molecule_logs/ # CI template for periodic CI jobs # Enabled when PERIODIC_CI_ENABLED var is set + .molecule_periodic: only: variables: - - $PERIODIC_CI_ENABLED + - $PERIODIC_CI_ENABLED allow_failure: true extends: .molecule @@ -34,26 +44,26 @@ molecule_full: molecule_no_container_engines: extends: .molecule script: - - ./tests/scripts/molecule_run.sh -e container-engine + - ./tests/scripts/molecule_run.sh -e container-engine when: on_success molecule_docker: extends: .molecule script: - - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd + - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd when: on_success molecule_containerd: extends: .molecule script: - - ./tests/scripts/molecule_run.sh -i container-engine/containerd + - ./tests/scripts/molecule_run.sh -i container-engine/containerd when: on_success molecule_cri-o: extends: .molecule stage: deploy-part2 script: - - ./tests/scripts/molecule_run.sh -i container-engine/cri-o + - ./tests/scripts/molecule_run.sh -i container-engine/cri-o allow_failure: true when: on_success @@ -63,7 +73,7 @@ molecule_kata: stage: deploy-part3 allow_failure: true script: - - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers + - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers when: on_success molecule_gvisor: @@ -71,7 +81,7 @@ molecule_gvisor: stage: deploy-part3 allow_failure: true script: - - ./tests/scripts/molecule_run.sh -i container-engine/gvisor + - ./tests/scripts/molecule_run.sh -i container-engine/gvisor when: on_success molecule_youki: @@ -79,5 +89,5 @@ molecule_youki: stage: deploy-part3 allow_failure: true script: - - ./tests/scripts/molecule_run.sh -i container-engine/youki + - ./tests/scripts/molecule_run.sh -i container-engine/youki when: on_success diff --git a/.gitlab-ci/terraform.yml b/.gitlab-ci/terraform.yml index 097ae0f0a..5e7fa6cda 100644 --- a/.gitlab-ci/terraform.yml +++ b/.gitlab-ci/terraform.yml @@ -51,7 +51,7 @@ - tests/scripts/testcases_run.sh after_script: # Cleanup regardless of exit code - - chronic ./tests/scripts/testcases_cleanup.sh + - ./tests/scripts/testcases_cleanup.sh tf-validate-openstack: extends: .terraform_validate diff --git a/.gitlab-ci/vagrant.yml b/.gitlab-ci/vagrant.yml index c0250681b..8fd03fd7b 100644 --- a/.gitlab-ci/vagrant.yml +++ b/.gitlab-ci/vagrant.yml @@ -1,5 +1,4 @@ --- - .vagrant: extends: .testcases variables: @@ -7,18 +6,25 @@ SSH_USER: "vagrant" VAGRANT_DEFAULT_PROVIDER: "libvirt" KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb - tags: [c3.small.x86] - only: [/^pr-.*$/] - except: ['triggers'] - image: $PIPELINE_IMAGE + DOCKER_NAME: vagrant + VAGRANT_ANSIBLE_TAGS: facts + tags: [ffci-vm-large] + # only: [/^pr-.*$/] + # except: ['triggers'] + image: quay.io/kubespray/vm-kubespray-ci:v6 services: [] before_script: + - echo $USER + - python3 -m venv citest + - source citest/bin/activate + - vagrant plugin expunge --reinstall --force --no-tty + - vagrant plugin install vagrant-libvirt + - pip install --no-compile --no-cache-dir pip -U + - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt + - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt - ./tests/scripts/vagrant_clean.sh script: - ./tests/scripts/testcases_run.sh - after_script: - - chronic ./tests/scripts/testcases_cleanup.sh - allow_failure: true vagrant_ubuntu20-calico-dual-stack: stage: deploy-part2 diff --git a/Vagrantfile b/Vagrantfile index 498737668..da2e0dee7 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -255,7 +255,9 @@ Vagrant.configure("2") do |config| "kubectl_localhost": "True", "local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}", "local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}", - "ansible_ssh_user": SUPPORTED_OS[$os][:user] + "ansible_ssh_user": SUPPORTED_OS[$os][:user], + "ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"), + "unsafe_show_logs": "True" } # Only execute the Ansible provisioner once, when all the machines are up and ready. diff --git a/pipeline.Dockerfile b/pipeline.Dockerfile index a43761b9f..ccd83637c 100644 --- a/pipeline.Dockerfile +++ b/pipeline.Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:jammy-20230308 # Pip needs this as well at the moment to install ansible # (and potentially other packages) # See: https://github.com/pypa/pip/issues/10219 -ENV VAGRANT_VERSION=2.3.7 \ +ENV VAGRANT_VERSION=2.4.1 \ VAGRANT_DEFAULT_PROVIDER=libvirt \ VAGRANT_ANSIBLE_TAGS=facts \ LANG=C.UTF-8 \ @@ -31,6 +31,8 @@ RUN apt update -q \ unzip \ libvirt-clients \ qemu-utils \ + qemu-kvm \ + dnsmasq \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ && add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ && apt update -q \ @@ -45,6 +47,7 @@ RUN --mount=type=bind,target=./requirements.txt,src=./requirements.txt \ update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ && pip install --no-compile --no-cache-dir pip -U \ && pip install --no-compile --no-cache-dir -r tests/requirements.txt \ + && pip install --no-compile --no-cache-dir -r requirements.txt \ && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \ && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ diff --git a/tests/Makefile b/tests/Makefile index c9f561eee..0b1eead01 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -83,8 +83,7 @@ cleanup-packet: create-vagrant: vagrant up - find / -name vagrant_ansible_inventory - cp /builds/kargo-ci/kubernetes-sigs-kubespray/inventory/sample/vagrant_ansible_inventory $(INVENTORY) + cp $(CI_PROJECT_DIR)/inventory/sample/vagrant_ansible_inventory $(INVENTORY) delete-vagrant: vagrant destroy -f diff --git a/tests/files/vagrant_ubuntu20-flannel.rb b/tests/files/vagrant_ubuntu20-flannel.rb index 55daa19e3..d8bf563a3 100644 --- a/tests/files/vagrant_ubuntu20-flannel.rb +++ b/tests/files/vagrant_ubuntu20-flannel.rb @@ -2,7 +2,6 @@ $os = "ubuntu2004" # For CI we are not worries about data persistence across reboot $libvirt_volume_cache = "unsafe" - # Checking for box update can trigger API rate limiting # https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html $box_check_update = false diff --git a/tests/files/vagrant_ubuntu20-flannel.yml b/tests/files/vagrant_ubuntu20-flannel.yml index 6f8916feb..43ec54c6a 100644 --- a/tests/files/vagrant_ubuntu20-flannel.yml +++ b/tests/files/vagrant_ubuntu20-flannel.yml @@ -1,3 +1,4 @@ --- # Kubespray settings kube_network_plugin: flannel +ansible_ssh_private_key: .vagrant.d/insecure_private_key diff --git a/tests/scripts/vagrant-install.sh b/tests/scripts/vagrant-install.sh new file mode 100755 index 000000000..a2d1f0a3c --- /dev/null +++ b/tests/scripts/vagrant-install.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# install_vagrant() { +# sudo apt install vagrant-libvirt vagrant -y +# sudo vagrant plugin install vagrant-libvirt +# } + +# prep(){ +# sudo apt-get update -y +# sudo apt-get install ca-certificates curl libvirt-daemon-system\ +# libvirt-clients qemu-utils qemu-kvm htop atop -y + +# sudo install -m 0755 -d /etc/apt/keyrings +# } +# install_docker() { +# VERSION_STRING=5:26.1.0-1~ubuntu.24.04~noble +# sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +# sudo chmod a+r /etc/apt/keyrings/docker.asc + +# # Add the repository to Apt sources: +# echo \ +# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ +# $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ +# sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +# sudo apt-get update -y + +# sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y +# } +# install_docker_auto () { +# curl -fsSL https://get.docker.com -o get-docker.sh +# sudo sh ./get-docker.sh --dry-run +# } + + + +VAGRANT_VERSION=2.4.1 +VAGRANT_DEFAULT_PROVIDER=libvirt +VAGRANT_ANSIBLE_TAGS=facts +LANG=C.UTF-8 +DEBIAN_FRONTEND=noninteractive +PYTHONDONTWRITEBYTECODE=1 +KUBE_VERSION=1.29.5 +pipeline_install() { + cp /etc/apt/sources.list /etc/apt/sources.list."$(date +"%F")" + sed -i -e '/^# deb-src.*universe$/s/# //g' /etc/apt/sources.list + sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources + + apt update + # libssl-dev \ + # python3-dev \ + # # jq \ + # moreutils \ + # libvirt-dev \ + # # rsync \ + # git \ + # # htop \ + # gpg \ + # atop + + # gnupg2 \ +# software-properties-common +# + apt install --no-install-recommends -y \ + git \ + make \ + python3-pip \ + sshpass \ + apt-transport-https \ + openssh-client \ + ca-certificates \ + curl \ + libfuse2 \ + unzip \ + qemu-utils \ + libvirt-daemon-system \ + libvirt-clients \ + qemu-kvm \ + ebtables libguestfs-tools \ + ruby-fog-libvirt \ + libvirt-dev \ + gcc \ + build-essential \ + ruby-libvirt \ + libxslt-dev libxml2-dev zlib1g-dev \ + python3-venv python3-full \ + dnsmasq + + apt-get build-dep -y ruby-libvirt ruby-dev + ### VAGRANT ### + # apt-get install -y unzip + curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_linux_amd64.zip + unzip vagrant_${VAGRANT_VERSION}_linux_amd64.zip + mv vagrant /usr/local/bin/vagrant + chmod a+x /usr/local/bin/vagrant + # ls -la /usr/local/bin/vagrant + /usr/local/bin/vagrant plugin install vagrant-libvirt + usermod -aG kvm kubespray + usermod -aG libvirt kubespray + + ### DOCKER ### + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository -y "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + apt update + apt install --no-install-recommends -y docker-ce + apt autoremove -y --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/* + + ### KUBECTL ### + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + mv kubectl /usr/local/bin/kubectl + chmod a+x /usr/local/bin/kubectl + systemctl restart libvirtd + # Install Vagrant + # apt update -y + # echo apt-get install -y unzip libfuse2 vagrant vagrant-libvirt + # apt --fix-broken install -y + # dpkg --configure -a -y + + +} +# wrapped up in a function so that we have some protection against only getting +# half the file during "curl | sh" +pipeline_install