tests: cleanup stale packet namespace automatically (#10245)
* tests: cleanup stale packet namespace automatically Cancelled job on Gitlab can produce stale VMs as the delete playbook will never be executed. This commits allow removing old vms by getting all the namespace created from the same branch with an older pipeline id. Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch> * tests: cleanup stale packet namespace after 2 hours This ensure that we don't have any packet namespace remaining for more than 2 hours. All the jobs complete usually within 30min-1hour so 2 hours is enough to detect a stale namespace. Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch> * tests: ignore vm cleanup failure Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch> * tests: use pipeline_id var instead of fetching namespace for cleanup packet vm Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch> --------- Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>pull/10256/head
parent
eb31653d66
commit
3311e0a296
|
@ -23,6 +23,14 @@
|
|||
allow_failure: true
|
||||
extends: .packet
|
||||
|
||||
packet_cleanup_old:
|
||||
stage: deploy-part1
|
||||
extends: .packet_periodic
|
||||
script:
|
||||
- cd tests
|
||||
- make cleanup-packet
|
||||
after_script: []
|
||||
|
||||
# The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken
|
||||
packet_ubuntu20-calico-aio:
|
||||
stage: deploy-part1
|
||||
|
|
|
@ -64,6 +64,8 @@ create-packet: init-packet
|
|||
$(ANSIBLE_LOG_LEVEL) \
|
||||
-e @"files/${CI_JOB_NAME}.yml" \
|
||||
-e test_id=$(TEST_ID) \
|
||||
-e branch="$(CI_COMMIT_BRANCH)" \
|
||||
-e pipeline_id="$(CI_PIPELINE_ID)" \
|
||||
-e inventory_path=$(INVENTORY)
|
||||
|
||||
delete-packet:
|
||||
|
@ -71,8 +73,14 @@ delete-packet:
|
|||
$(ANSIBLE_LOG_LEVEL) \
|
||||
-e @"files/${CI_JOB_NAME}.yml" \
|
||||
-e test_id=$(TEST_ID) \
|
||||
-e branch="$(CI_COMMIT_BRANCH)" \
|
||||
-e pipeline_id="$(CI_PIPELINE_ID)" \
|
||||
-e inventory_path=$(INVENTORY)
|
||||
|
||||
cleanup-packet:
|
||||
ansible-playbook cloud_playbooks/cleanup-packet.yml -c local \
|
||||
$(ANSIBLE_LOG_LEVEL)
|
||||
|
||||
create-vagrant:
|
||||
vagrant up
|
||||
find / -name vagrant_ansible_inventory
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
|
||||
- hosts: localhost
|
||||
gather_facts: no
|
||||
become: true
|
||||
roles:
|
||||
- { role: cleanup-packet-ci }
|
|
@ -0,0 +1,16 @@
|
|||
---
|
||||
|
||||
- name: Fetch a list of namespaces
|
||||
kubernetes.core.k8s_info:
|
||||
api_version: v1
|
||||
kind: Namespace
|
||||
label_selectors:
|
||||
- cijobs = true
|
||||
register: namespaces
|
||||
|
||||
- name: Delete stale namespaces for more than 2 hours
|
||||
command: "kubectl delete namespace {{ item.metadata.name }}"
|
||||
failed_when: false
|
||||
loop: "{{ namespaces.resources }}"
|
||||
when:
|
||||
- (now() - (item.metadata.creationTimestamp | to_datetime("%Y-%m-%dT%H:%M:%SZ"))).total_seconds() >= 7200
|
|
@ -0,0 +1,17 @@
|
|||
---
|
||||
|
||||
- name: Fetch a list of namespaces
|
||||
kubernetes.core.k8s_info:
|
||||
api_version: v1
|
||||
kind: Namespace
|
||||
label_selectors:
|
||||
- cijobs = true
|
||||
- branch = {{ branch }}
|
||||
register: namespaces
|
||||
|
||||
- name: Delete older namespaces
|
||||
command: "kubectl delete namespace {{ item.metadata.name }}"
|
||||
failed_when: false
|
||||
loop: "{{ namespaces.resources }}"
|
||||
when:
|
||||
- (item.metadata.labels.pipeline_id | int) < (pipeline_id | int)
|
|
@ -1,7 +1,9 @@
|
|||
---
|
||||
|
||||
- name: "Create CI namespace {{ test_name }} for test vms"
|
||||
command: "kubectl create namespace {{ test_name }}"
|
||||
shell: |-
|
||||
kubectl create namespace {{ test_name }} &&
|
||||
kubectl label namespace {{ test_name }} cijobs=true branch="{{ branch }}" pipeline_id="{{ pipeline_id }}"
|
||||
changed_when: false
|
||||
|
||||
- name: "Create temp dir /tmp/{{ test_name }} for CI files"
|
||||
|
|
|
@ -7,6 +7,8 @@
|
|||
set_fact:
|
||||
vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale', 'ha-recover', 'ha-recover-noquorum'] -%}{{ 3|int }}{%- elif mode == 'aio' -%}{{ 1|int }}{%- else -%}{{ 2|int }}{%- endif -%}"
|
||||
|
||||
- import_tasks: cleanup-old-vms.yml
|
||||
|
||||
- import_tasks: create-vms.yml
|
||||
when:
|
||||
- not vm_cleanup
|
||||
|
|
Loading…
Reference in New Issue