Merge pull request #149 from thedac/timeout-error

Explicit Model Timeout Exception

unit_tests/test_zaza_model.py

@@ -14,6 +14,7 @@
 
 import aiounittest
 import asyncio.futures
+import concurrent
 import mock
 
 import unit_tests.utils as ut_utils
@@ -386,8 +387,14 @@ class TestModel(ut_utils.BaseTestCase):
 
     def _application_states_setup(self, setup, units_idle=True):
         self.system_ready = True
+        self._block_until_calls = 0
 
-        async def _block_until(f, timeout=None):
+        async def _block_until(f, timeout=0):
+            # Mimic timeouts
+            timeout = timeout + self._block_until_calls
+            self._block_until_calls += 1
+            if timeout == -1:
+                raise concurrent.futures._base.TimeoutError("Timeout", 1)
             result = f()
             if not result:
                 self.system_ready = False
@@ -539,7 +546,7 @@ class TestModel(ut_utils.BaseTestCase):
             timeout=1)
         self.assertTrue(self.system_ready)
 
-    def test_wait_for_application_states_bespoke_msg_bloked_ok(self):
+    def test_wait_for_application_states_bespoke_msg_blocked_ok(self):
         self._application_states_setup({
             'workload-status': 'blocked',
             'workload-status-message': 'Sure, I could do something'})
@@ -551,6 +558,29 @@ class TestModel(ut_utils.BaseTestCase):
             timeout=1)
         self.assertTrue(self.system_ready)
 
+    def test_wait_for_application_states_idle_timeout(self):
+        self._application_states_setup({
+            'agent-status': 'executing',
+            'workload-status': 'blocked',
+            'workload-status-message': 'Sure, I could do something'})
+        with self.assertRaises(model.ModelTimeout) as timeout:
+            model.wait_for_application_states('modelname', timeout=-2)
+        self.assertEqual(
+            timeout.exception.args[0],
+            "Zaza has timed out waiting on the model to reach idle state.")
+
+    def test_wait_for_application_states_timeout(self):
+        self._application_states_setup({
+            'agent-status': 'executing',
+            'workload-status': 'blocked',
+            'workload-status-message': 'Sure, I could do something'})
+        with self.assertRaises(model.ModelTimeout) as timeout:
+            model.wait_for_application_states('modelname', timeout=-3)
+        self.assertEqual(
+            timeout.exception.args[0],
+            "Zaza has timed out waiting on the model to reach expected "
+            "workload statuses.")
+
     def test_get_current_model(self):
         self.patch_object(model, 'Model')
         self.Model.return_value = self.Model_mock

zaza/model.py

@@ -27,6 +27,7 @@ import subprocess
 import tempfile
 import yaml
 from oslo_config import cfg
+import concurrent
 
 from juju.errors import JujuError
 from juju.model import Model
@@ -36,6 +37,12 @@ from zaza import sync_wrapper
 CURRENT_MODEL = None
 
 
+class ModelTimeout(Exception):
+    """Model timeout exception."""
+
+    pass
+
+
 def set_juju_model(model_name):
     """Point environment at the given model.
 
@@ -725,32 +732,40 @@ async def async_wait_for_application_states(model_name=None, states=None,
             lambda: len(model.units) > 0
         )
         logging.info("Waiting for all units to be idle")
-        await model.block_until(
-            lambda: model.all_units_idle(), timeout=timeout)
-        for application in model.applications:
-            check_info = states.get(application, {})
-            for unit in model.applications[application].units:
-                logging.info("Checking workload status of {}".format(
-                    unit.entity_id))
-                await model.block_until(
-                    lambda: check_unit_workload_status(
-                        model,
-                        unit,
-                        check_info.get('workload-status', 'active')),
-                    timeout=timeout)
-                check_msg = check_info.get('workload-status-message')
-                logging.info("Checking workload status message of {}".format(
-                    unit.entity_id))
-                if check_msg is not None:
-                    prefixes = (check_msg)
-                else:
-                    prefixes = approved_message_prefixes
-                await model.block_until(
-                    lambda: check_unit_workload_status_message(
-                        model,
-                        unit,
-                        prefixes=prefixes),
-                    timeout=timeout)
+        try:
+            await model.block_until(
+                lambda: model.all_units_idle(), timeout=timeout)
+        except concurrent.futures._base.TimeoutError:
+            raise ModelTimeout("Zaza has timed out waiting on the model to "
+                               "reach idle state.")
+        try:
+            for application, app_data in model.applications.items():
+                check_info = states.get(application, {})
+                for unit in app_data.units:
+                    logging.info("Checking workload status of {}".format(
+                        unit.entity_id))
+                    await model.block_until(
+                        lambda: check_unit_workload_status(
+                            model,
+                            unit,
+                            check_info.get('workload-status', 'active')),
+                        timeout=timeout)
+                    check_msg = check_info.get('workload-status-message')
+                    logging.info("Checking workload status message of {}"
+                                 .format(unit.entity_id))
+                    if check_msg is not None:
+                        prefixes = (check_msg)
+                    else:
+                        prefixes = approved_message_prefixes
+                    await model.block_until(
+                        lambda: check_unit_workload_status_message(
+                            model,
+                            unit,
+                            prefixes=prefixes),
+                        timeout=timeout)
+        except concurrent.futures._base.TimeoutError:
+            raise ModelTimeout("Zaza has timed out waiting on the model to "
+                               "reach expected workload statuses.")
 
 wait_for_application_states = sync_wrapper(async_wait_for_application_states)