From 769e0d2f5c5583791b60c8210ce6c598d61b29a9 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 22 May 2019 10:42:33 +0200 Subject: [PATCH] tests: add retries on failing tests in testinfra This commit adds `pytest-rerunfailures` in requirements.txt so we can retry failing test in testinfra to avoid false positive. (eg: sometimes it can happen for some reason a service takes too much time to start) Signed-off-by: Guillaume Abrioux (cherry picked from commit 4708b7615f6acab9fe9f251eaf2a6da2e1f859ab) --- tests/requirements.txt | 1 + tox-dashboard.ini | 2 +- tox-update.ini | 2 +- tox.ini | 26 +++++++++++++------------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index bd4727924..ab75cb11a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,3 +9,4 @@ netaddr mock jmespath paramiko +pytest-rerunfailures \ No newline at end of file diff --git a/tox-dashboard.ini b/tox-dashboard.ini index def8690a3..29cb7b51f 100644 --- a/tox-dashboard.ini +++ b/tox-dashboard.ini @@ -64,6 +64,6 @@ commands= ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \ " - bash -c "CEPH_STABLE_RELEASE={env:CEPH_STABLE_RELEASE:nautilus} py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" + bash -c "CEPH_STABLE_RELEASE={env:CEPH_STABLE_RELEASE:nautilus} py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" vagrant destroy --force diff --git a/tox-update.ini b/tox-update.ini index 9374cba30..41f5d639d 100644 --- a/tox-update.ini +++ b/tox-update.ini @@ -76,6 +76,6 @@ commands= ceph_docker_image_tag={env:UPDATE_CEPH_DOCKER_IMAGE_TAG:latest-nautilus} \ " - bash -c "CEPH_STABLE_RELEASE={env:UPDATE_CEPH_STABLE_RELEASE:nautilus} py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" + bash -c "CEPH_STABLE_RELEASE={env:UPDATE_CEPH_STABLE_RELEASE:nautilus} py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" vagrant destroy --force diff --git a/tox.ini b/tox.ini index 70fde3014..2113ff44d 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,7 @@ commands= # wait 30sec for services to be ready sleep 30 # test cluster state using ceph-ansible tests - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {envdir}/tmp/ceph-ansible/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {envdir}/tmp/ceph-ansible/tests/functional/tests # install ceph-ansible@master requirements pip install -r {toxinidir}/tests/requirements.txt @@ -68,13 +68,13 @@ commands= " # test cluster state again using ceph-ansible tests - bash -c "CEPH_STABLE_RELEASE=nautilus py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" + bash -c "CEPH_STABLE_RELEASE=nautilus py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" # reboot all vms ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/reboot.yml # retest to ensure cluster came back up correctly after rebooting - bash -c "CEPH_STABLE_RELEASE=nautilus py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" + bash -c "CEPH_STABLE_RELEASE=nautilus py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" vagrant destroy --force @@ -136,7 +136,7 @@ commands= ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG:latest-master} \ " # test that the cluster can be redeployed in a healthy state - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [purge-lvm] commands= @@ -161,7 +161,7 @@ commands= ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG:latest-master} \ " # test that the cluster can be redeployed in a healthy state - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [shrink-mon] commands= @@ -190,7 +190,7 @@ commands= ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG:latest-master} \ " - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-switch-to-containers --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-switch-to-containers --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-mons] commands= @@ -203,7 +203,7 @@ commands= ceph_docker_image={env:CEPH_DOCKER_IMAGE:ceph/daemon} \ ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG:latest-nautilus} \ " - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-osds] commands= @@ -217,7 +217,7 @@ commands= ceph_docker_image={env:CEPH_DOCKER_IMAGE:ceph/daemon} \ ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG:latest-master} \ " - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-mgrs] commands= @@ -247,7 +247,7 @@ commands= ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \ ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " - py.test -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-rbdmirrors] commands= @@ -262,7 +262,7 @@ commands= ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \ ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " - py.test -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-rgws] commands= @@ -277,7 +277,7 @@ commands= ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \ ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " - py.test -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [rgw-multisite] commands= @@ -413,14 +413,14 @@ commands= # wait 30sec for services to be ready sleep 30 # test cluster state using ceph-ansible tests - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests # reboot all vms all_daemons: ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/reboot.yml # wait 30sec for services to be ready # retest to ensure cluster came back up correctly after rebooting - all_daemons: py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + all_daemons: py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests # handlers/idempotency test all_daemons: ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars "delegate_facts_host={env:DELEGATE_FACTS_HOST:True} fetch_directory={env:FETCH_DIRECTORY:{changedir}/fetch} ceph_stable_release={env:CEPH_STABLE_RELEASE:nautilus} ceph_docker_registry={env:CEPH_DOCKER_REGISTRY:docker.io} ceph_docker_image={env:CEPH_DOCKER_IMAGE:ceph/daemon} ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG_BIS:latest-bis-master} copy_admin_key={env:COPY_ADMIN_KEY:False} " --extra-vars @ceph-override.json