diff --git a/unit_tests/test_zaza_model.py b/unit_tests/test_zaza_model.py index 2b80080..c8d0ec2 100644 --- a/unit_tests/test_zaza_model.py +++ b/unit_tests/test_zaza_model.py @@ -14,6 +14,7 @@ import aiounittest import asyncio.futures +import concurrent import mock import unit_tests.utils as ut_utils @@ -386,8 +387,14 @@ class TestModel(ut_utils.BaseTestCase): def _application_states_setup(self, setup, units_idle=True): self.system_ready = True + self._block_until_calls = 0 - async def _block_until(f, timeout=None): + async def _block_until(f, timeout=0): + # Mimic timeouts + timeout = timeout + self._block_until_calls + self._block_until_calls += 1 + if timeout == -1: + raise concurrent.futures._base.TimeoutError("Timeout", 1) result = f() if not result: self.system_ready = False @@ -539,7 +546,7 @@ class TestModel(ut_utils.BaseTestCase): timeout=1) self.assertTrue(self.system_ready) - def test_wait_for_application_states_bespoke_msg_bloked_ok(self): + def test_wait_for_application_states_bespoke_msg_blocked_ok(self): self._application_states_setup({ 'workload-status': 'blocked', 'workload-status-message': 'Sure, I could do something'}) @@ -551,6 +558,29 @@ class TestModel(ut_utils.BaseTestCase): timeout=1) self.assertTrue(self.system_ready) + def test_wait_for_application_states_idle_timeout(self): + self._application_states_setup({ + 'agent-status': 'executing', + 'workload-status': 'blocked', + 'workload-status-message': 'Sure, I could do something'}) + with self.assertRaises(model.ModelTimeout) as timeout: + model.wait_for_application_states('modelname', timeout=-2) + self.assertEqual( + timeout.exception.args[0], + "Zaza has timed out waiting on the model to reach idle state.") + + def test_wait_for_application_states_timeout(self): + self._application_states_setup({ + 'agent-status': 'executing', + 'workload-status': 'blocked', + 'workload-status-message': 'Sure, I could do something'}) + with self.assertRaises(model.ModelTimeout) as timeout: + model.wait_for_application_states('modelname', timeout=-3) + self.assertEqual( + timeout.exception.args[0], + "Zaza has timed out waiting on the model to reach expected " + "workload statuses.") + def test_get_current_model(self): self.patch_object(model, 'Model') self.Model.return_value = self.Model_mock diff --git a/zaza/model.py b/zaza/model.py index 8a3efff..dc391b1 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -27,6 +27,7 @@ import subprocess import tempfile import yaml from oslo_config import cfg +import concurrent from juju.errors import JujuError from juju.model import Model @@ -36,6 +37,12 @@ from zaza import sync_wrapper CURRENT_MODEL = None +class ModelTimeout(Exception): + """Model timeout exception.""" + + pass + + def set_juju_model(model_name): """Point environment at the given model. @@ -725,32 +732,40 @@ async def async_wait_for_application_states(model_name=None, states=None, lambda: len(model.units) > 0 ) logging.info("Waiting for all units to be idle") - await model.block_until( - lambda: model.all_units_idle(), timeout=timeout) - for application in model.applications: - check_info = states.get(application, {}) - for unit in model.applications[application].units: - logging.info("Checking workload status of {}".format( - unit.entity_id)) - await model.block_until( - lambda: check_unit_workload_status( - model, - unit, - check_info.get('workload-status', 'active')), - timeout=timeout) - check_msg = check_info.get('workload-status-message') - logging.info("Checking workload status message of {}".format( - unit.entity_id)) - if check_msg is not None: - prefixes = (check_msg) - else: - prefixes = approved_message_prefixes - await model.block_until( - lambda: check_unit_workload_status_message( - model, - unit, - prefixes=prefixes), - timeout=timeout) + try: + await model.block_until( + lambda: model.all_units_idle(), timeout=timeout) + except concurrent.futures._base.TimeoutError: + raise ModelTimeout("Zaza has timed out waiting on the model to " + "reach idle state.") + try: + for application, app_data in model.applications.items(): + check_info = states.get(application, {}) + for unit in app_data.units: + logging.info("Checking workload status of {}".format( + unit.entity_id)) + await model.block_until( + lambda: check_unit_workload_status( + model, + unit, + check_info.get('workload-status', 'active')), + timeout=timeout) + check_msg = check_info.get('workload-status-message') + logging.info("Checking workload status message of {}" + .format(unit.entity_id)) + if check_msg is not None: + prefixes = (check_msg) + else: + prefixes = approved_message_prefixes + await model.block_until( + lambda: check_unit_workload_status_message( + model, + unit, + prefixes=prefixes), + timeout=timeout) + except concurrent.futures._base.TimeoutError: + raise ModelTimeout("Zaza has timed out waiting on the model to " + "reach expected workload statuses.") wait_for_application_states = sync_wrapper(async_wait_for_application_states)