diff --git a/tests/requirements.txt b/tests/requirements.txt index bd4727924..ab75cb11a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,3 +9,4 @@ netaddr mock jmespath paramiko +pytest-rerunfailures \ No newline at end of file diff --git a/tox-dashboard.ini b/tox-dashboard.ini index def8690a3..29cb7b51f 100644 --- a/tox-dashboard.ini +++ b/tox-dashboard.ini @@ -64,6 +64,6 @@ commands= ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \ " - bash -c "CEPH_STABLE_RELEASE={env:CEPH_STABLE_RELEASE:nautilus} py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" + bash -c "CEPH_STABLE_RELEASE={env:CEPH_STABLE_RELEASE:nautilus} py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" vagrant destroy --force diff --git a/tox-update.ini b/tox-update.ini index 0ef2f3c07..b3bf7e762 100644 --- a/tox-update.ini +++ b/tox-update.ini @@ -80,6 +80,6 @@ commands= ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \ " - bash -c "CEPH_STABLE_RELEASE={env:UPDATE_CEPH_STABLE_RELEASE:nautilus} py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" + bash -c "CEPH_STABLE_RELEASE={env:UPDATE_CEPH_STABLE_RELEASE:nautilus} py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" vagrant destroy --force diff --git a/tox.ini b/tox.ini index a716ad5bc..9d40657d0 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,7 @@ commands= # wait 30sec for services to be ready sleep 30 # test cluster state using ceph-ansible tests - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {envdir}/tmp/ceph-ansible/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {envdir}/tmp/ceph-ansible/tests/functional/tests # install ceph-ansible@master requirements pip install -r {toxinidir}/tests/requirements.txt @@ -68,13 +68,13 @@ commands= " # test cluster state again using ceph-ansible tests - bash -c "CEPH_STABLE_RELEASE=nautilus py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" + bash -c "CEPH_STABLE_RELEASE=nautilus py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" # reboot all vms ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/reboot.yml # retest to ensure cluster came back up correctly after rebooting - bash -c "CEPH_STABLE_RELEASE=nautilus py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" + bash -c "CEPH_STABLE_RELEASE=nautilus py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests" vagrant destroy --force @@ -138,7 +138,7 @@ commands= ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " # test that the cluster can be redeployed in a healthy state - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [purge-lvm] commands= @@ -165,7 +165,7 @@ commands= ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " # test that the cluster can be redeployed in a healthy state - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [shrink-mon] commands= @@ -196,7 +196,7 @@ commands= ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-switch-to-containers --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-switch-to-containers --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-mons] commands= @@ -211,7 +211,7 @@ commands= ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \ ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-osds] commands= @@ -227,7 +227,7 @@ commands= ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \ ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-mgrs] commands= @@ -257,7 +257,7 @@ commands= ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \ ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " - py.test -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-rbdmirrors] commands= @@ -272,7 +272,7 @@ commands= ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \ ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " - py.test -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [add-rgws] commands= @@ -287,7 +287,7 @@ commands= ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \ ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \ " - py.test -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests [rgw-multisite] commands= @@ -434,14 +434,14 @@ commands= # wait 30sec for services to be ready sleep 30 # test cluster state using ceph-ansible tests - py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests # reboot all vms all_daemons: ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/reboot.yml # wait 30sec for services to be ready # retest to ensure cluster came back up correctly after rebooting - all_daemons: py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests + all_daemons: py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests # handlers/idempotency test all_daemons: ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars "delegate_facts_host={env:DELEGATE_FACTS_HOST:True} fetch_directory={env:FETCH_DIRECTORY:{changedir}/fetch} ceph_stable_release={env:CEPH_STABLE_RELEASE:nautilus} ceph_docker_registry={env:CEPH_DOCKER_REGISTRY:docker.io} ceph_docker_image={env:CEPH_DOCKER_IMAGE:ceph/daemon} ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG_BIS:latest-bis-master} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} copy_admin_key={env:COPY_ADMIN_KEY:False}" --extra-vars @ceph-override.json