diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 4044c55..5fbfb7e 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -19,7 +19,7 @@ import logging import os import re import tempfile -import time +import tenacity import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.model @@ -477,6 +477,23 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): states=test_config.get("target_deploy_status", {})) +@tenacity.retry( + retry=tenacity.retry_if_result(lambda is_new: is_new is False), + wait=tenacity.wait_fixed(5), # interval between retries + stop=tenacity.stop_after_attempt(10)) # retry times +def retry_is_new_crm_master(test, old_crm_master): + """Check new crm master with retries. + + Return True if a new crm master detected, retry 10 times if False. + """ + new_crm_master = test.get_crm_master() + if new_crm_master and new_crm_master != old_crm_master: + logging.info( + "New crm_master unit detected on {}".format(new_crm_master)) + return True + return False + + class PerconaClusterScaleTests(PerconaClusterBaseTest): """Percona Cluster scale tests.""" @@ -496,22 +513,9 @@ class PerconaClusterScaleTests(PerconaClusterBaseTest): zaza.model.run_on_unit(old_crm_master, cmd) logging.info("looking for the new crm_master") - i = 0 - while i < 10: - i += 1 - # XXX time.sleep roundup - # https://github.com/openstack-charmers/zaza-openstack-tests/issues/46 - time.sleep(5) # give some time to pacemaker to react - new_crm_master = self.get_crm_master() - - if (new_crm_master and new_crm_master != old_crm_master): - logging.info( - "New crm_master unit detected" - " on {}".format(new_crm_master) - ) - break - else: - assert False, "The crm_master didn't change" + self.assertTrue( + retry_is_new_crm_master(self, old_crm_master), + msg="The crm_master didn't change") # Check connectivity on the VIP # \ is required due to pep8 and parenthesis would make the assertion diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 9b898bf..a83440f 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -17,16 +17,16 @@ """Collection of tests for vault.""" import contextlib -import hvac import json import logging -import time import unittest import uuid import tempfile -import tenacity import requests +import tenacity +from hvac.exceptions import InternalServerError + import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.charm_tests.vault.utils as vault_utils @@ -36,6 +36,21 @@ import zaza.model import zaza.utilities.juju as juju_utils +@tenacity.retry( + retry=tenacity.retry_if_exception_type(InternalServerError), + retry_error_callback=lambda retry_state: False, + wait=tenacity.wait_fixed(2), # interval between retries + stop=tenacity.stop_after_attempt(10)) # retry 10 times +def retry_hvac_client_authenticated(client): + """Check hvac client is authenticated with retry. + + If is_authenticated() raise exception for all retries, + return False(which is done by `retry_error_callback`). + Otherwise, return whatever the returned value. + """ + return client.hvac_client.is_authenticated() + + class BaseVaultTest(test_utils.OpenStackBaseTest): """Base class for vault tests.""" @@ -189,17 +204,7 @@ class VaultTest(BaseVaultTest): def test_all_clients_authenticated(self): """Check all vault clients are authenticated.""" for client in self.clients: - for i in range(1, 10): - try: - self.assertTrue(client.hvac_client.is_authenticated()) - except hvac.exceptions.InternalServerError: - # XXX time.sleep roundup - # https://github.com/openstack-charmers/zaza-openstack-tests/issues/46 - time.sleep(2) - else: - break - else: - self.assertTrue(client.hvac_client.is_authenticated()) + self.assertTrue(retry_hvac_client_authenticated(client)) def check_read(self, key, value): """Check reading the key from all vault units.""" diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 40e33ac..b4b5579 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -20,9 +20,9 @@ import base64 import hvac import requests import tempfile -import time import urllib3 import yaml +import tenacity import collections @@ -240,6 +240,15 @@ def extract_lead_unit_client( .format(application_name)) +@tenacity.retry( + retry=tenacity.retry_if_exception_type(( + ConnectionRefusedError, + urllib3.exceptions.NewConnectionError, + urllib3.exceptions.MaxRetryError, + requests.exceptions.ConnectionError)), + reraise=True, # if all retries failed, reraise the last exception + wait=tenacity.wait_fixed(2), # interval between retries + stop=tenacity.stop_after_attempt(10)) # retry 10 times def is_initialized(client): """Check if vault is initialized. @@ -247,23 +256,10 @@ def is_initialized(client): :type client: CharmVaultClient :returns: Whether vault is initialized :rtype: bool + + Raise the last exception if no value returned after retries. """ - initialized = False - for i in range(1, 10): - try: - initialized = client.hvac_client.is_initialized() - except (ConnectionRefusedError, - urllib3.exceptions.NewConnectionError, - urllib3.exceptions.MaxRetryError, - requests.exceptions.ConnectionError): - # XXX time.sleep roundup - # https://github.com/openstack-charmers/zaza-openstack-tests/issues/46 - time.sleep(2) - else: - break - else: - raise Exception("Cannot connect") - return initialized + return client.hvac_client.is_initialized() def ensure_secret_backend(client):