tests: cleanup stale packet namespace automatically (#10245)

* tests: cleanup stale packet namespace automatically

Cancelled job on Gitlab can produce stale VMs as the delete playbook
will never be executed. This commits allow removing old vms by getting
all the namespace created from the same branch with an older pipeline
id.

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* tests: cleanup stale packet namespace after 2 hours

This ensure that we don't have any packet namespace remaining for more
than 2 hours. All the jobs complete usually within 30min-1hour so 2
hours is enough to detect a stale namespace.

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* tests: ignore vm cleanup failure

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* tests: use pipeline_id var instead of fetching namespace for cleanup packet vm

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

---------

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>
pull/10256/head
Arthur Outhenin-Chalandre 2023-06-26 09:57:08 +02:00 committed by GitHub
parent eb31653d66
commit 3311e0a296
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 61 additions and 1 deletions

View File

@ -23,6 +23,14 @@
allow_failure: true allow_failure: true
extends: .packet extends: .packet
packet_cleanup_old:
stage: deploy-part1
extends: .packet_periodic
script:
- cd tests
- make cleanup-packet
after_script: []
# The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken # The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken
packet_ubuntu20-calico-aio: packet_ubuntu20-calico-aio:
stage: deploy-part1 stage: deploy-part1

View File

@ -64,6 +64,8 @@ create-packet: init-packet
$(ANSIBLE_LOG_LEVEL) \ $(ANSIBLE_LOG_LEVEL) \
-e @"files/${CI_JOB_NAME}.yml" \ -e @"files/${CI_JOB_NAME}.yml" \
-e test_id=$(TEST_ID) \ -e test_id=$(TEST_ID) \
-e branch="$(CI_COMMIT_BRANCH)" \
-e pipeline_id="$(CI_PIPELINE_ID)" \
-e inventory_path=$(INVENTORY) -e inventory_path=$(INVENTORY)
delete-packet: delete-packet:
@ -71,8 +73,14 @@ delete-packet:
$(ANSIBLE_LOG_LEVEL) \ $(ANSIBLE_LOG_LEVEL) \
-e @"files/${CI_JOB_NAME}.yml" \ -e @"files/${CI_JOB_NAME}.yml" \
-e test_id=$(TEST_ID) \ -e test_id=$(TEST_ID) \
-e branch="$(CI_COMMIT_BRANCH)" \
-e pipeline_id="$(CI_PIPELINE_ID)" \
-e inventory_path=$(INVENTORY) -e inventory_path=$(INVENTORY)
cleanup-packet:
ansible-playbook cloud_playbooks/cleanup-packet.yml -c local \
$(ANSIBLE_LOG_LEVEL)
create-vagrant: create-vagrant:
vagrant up vagrant up
find / -name vagrant_ansible_inventory find / -name vagrant_ansible_inventory

View File

@ -0,0 +1,7 @@
---
- hosts: localhost
gather_facts: no
become: true
roles:
- { role: cleanup-packet-ci }

View File

@ -0,0 +1,16 @@
---
- name: Fetch a list of namespaces
kubernetes.core.k8s_info:
api_version: v1
kind: Namespace
label_selectors:
- cijobs = true
register: namespaces
- name: Delete stale namespaces for more than 2 hours
command: "kubectl delete namespace {{ item.metadata.name }}"
failed_when: false
loop: "{{ namespaces.resources }}"
when:
- (now() - (item.metadata.creationTimestamp | to_datetime("%Y-%m-%dT%H:%M:%SZ"))).total_seconds() >= 7200

View File

@ -0,0 +1,17 @@
---
- name: Fetch a list of namespaces
kubernetes.core.k8s_info:
api_version: v1
kind: Namespace
label_selectors:
- cijobs = true
- branch = {{ branch }}
register: namespaces
- name: Delete older namespaces
command: "kubectl delete namespace {{ item.metadata.name }}"
failed_when: false
loop: "{{ namespaces.resources }}"
when:
- (item.metadata.labels.pipeline_id | int) < (pipeline_id | int)

View File

@ -1,7 +1,9 @@
--- ---
- name: "Create CI namespace {{ test_name }} for test vms" - name: "Create CI namespace {{ test_name }} for test vms"
command: "kubectl create namespace {{ test_name }}" shell: |-
kubectl create namespace {{ test_name }} &&
kubectl label namespace {{ test_name }} cijobs=true branch="{{ branch }}" pipeline_id="{{ pipeline_id }}"
changed_when: false changed_when: false
- name: "Create temp dir /tmp/{{ test_name }} for CI files" - name: "Create temp dir /tmp/{{ test_name }} for CI files"

View File

@ -7,6 +7,8 @@
set_fact: set_fact:
vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale', 'ha-recover', 'ha-recover-noquorum'] -%}{{ 3|int }}{%- elif mode == 'aio' -%}{{ 1|int }}{%- else -%}{{ 2|int }}{%- endif -%}" vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale', 'ha-recover', 'ha-recover-noquorum'] -%}{{ 3|int }}{%- elif mode == 'aio' -%}{{ 1|int }}{%- else -%}{{ 2|int }}{%- endif -%}"
- import_tasks: cleanup-old-vms.yml
- import_tasks: create-vms.yml - import_tasks: create-vms.yml
when: when:
- not vm_cleanup - not vm_cleanup