From a0ed15a68cf425abd15ff29b860056eb71509ea7 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sun, 14 Apr 2019 00:05:19 +0200 Subject: [PATCH] Detect units in ``error`` state while awaiting idle model At present Zaza will wait until it hits a timeout if one of the units enters a error state while it awaits a idle model. Add check for errored units to ``block_until_all_units_idle()`` and ``wait_for_application_states()`` Fixes #100 --- unit_tests/test_zaza_model.py | 27 +++++++++++++++++++++++++++ zaza/model.py | 14 ++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_zaza_model.py b/unit_tests/test_zaza_model.py index 0128b58..631daf0 100644 --- a/unit_tests/test_zaza_model.py +++ b/unit_tests/test_zaza_model.py @@ -527,6 +527,14 @@ class TestModel(ut_utils.BaseTestCase): model.wait_for_application_states('modelname', timeout=1) self.assertFalse(self.system_ready) + def test_wait_for_application_states_errored_unit(self): + self._application_states_setup({ + 'workload-status': 'error', + 'workload-status-message': 'Unit is ready'}) + with self.assertRaises(model.UnitError): + model.wait_for_application_states('modelname', timeout=1) + self.assertFalse(self.system_ready) + def test_wait_for_application_states_not_ready_wsmsg(self): self._application_states_setup({ 'workload-status': 'active', @@ -666,6 +674,25 @@ class TestModel(ut_utils.BaseTestCase): with self.assertRaises(asyncio.futures.TimeoutError): model.block_until_all_units_idle('modelname') + def test_async_block_until_all_units_idle_errored_unit(self): + + async def _block_until(f, timeout=None): + if not f(): + raise asyncio.futures.TimeoutError + + def _all_units_idle(): + return True + self.patch_object(model, 'Model') + self.Model.return_value = self.Model_mock + self.Model_mock.all_units_idle.side_effect = _all_units_idle + self.patch_object(model, 'units_with_wl_status_state') + unit = mock.MagicMock() + unit.entity_id = 'aerroredunit/0' + self.units_with_wl_status_state.return_value = [unit] + self.Model_mock.block_until.side_effect = _block_until + with self.assertRaises(model.UnitError): + model.block_until_all_units_idle('modelname') + def block_until_service_status_base(self, rou_return): async def _block_until(f, timeout=None): diff --git a/zaza/model.py b/zaza/model.py index 5685d54..42c7cf4 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -758,10 +758,15 @@ async def async_wait_for_application_states(model_name=None, states=None, logging.info("Waiting for all units to be idle") try: await model.block_until( - lambda: model.all_units_idle(), timeout=timeout) + lambda: units_with_wl_status_state( + model, 'error') or model.all_units_idle(), + timeout=timeout) except concurrent.futures._base.TimeoutError: raise ModelTimeout("Zaza has timed out waiting on the model to " "reach idle state.") + errored_units = units_with_wl_status_state(model, 'error') + if errored_units: + raise UnitError(errored_units) try: for application, app_data in model.applications.items(): check_info = states.get(application, {}) @@ -814,7 +819,12 @@ async def async_block_until_all_units_idle(model_name=None, timeout=2700): """ async with run_in_model(model_name) as model: await model.block_until( - lambda: model.all_units_idle(), timeout=timeout) + lambda: units_with_wl_status_state( + model, 'error') or model.all_units_idle(), + timeout=timeout) + errored_units = units_with_wl_status_state(model, 'error') + if errored_units: + raise UnitError(errored_units) block_until_all_units_idle = sync_wrapper(async_block_until_all_units_idle)