Expose resource_removeds retry options (#11)

resource_removed is used in many places to monitor removal of
different kinds of OpenStack resource. It is reasonable to expect a
flavor to be removed in a few seconds where as an image based instance
may take minutes to be removed. With that in mind this change
exposes the retry options used by tenacity allowing the caller to
set reasonable expectations for the resource removal.

unit_tests/utilities/test_zaza_utilities_openstack.py

@@ -406,18 +406,60 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
             wait_exponential_multiplier=2,
             wait_iteration_max_time=20)
 
-    def test_resource_removed(self):
+    def test__resource_removed(self):
         resource_mock = mock.MagicMock()
         resource_mock.list.return_value = [mock.MagicMock(id='ba8204b0')]
-        openstack_utils.resource_removed(resource_mock, 'e01df65a')
+        openstack_utils._resource_removed(resource_mock, 'e01df65a')
 
-    def test_resource_removed_fail(self):
-        openstack_utils.resource_removed.retry.wait = \
-            tenacity.wait_none()
+    def test__resource_removed_fail(self):
         resource_mock = mock.MagicMock()
         resource_mock.list.return_value = [mock.MagicMock(id='e01df65a')]
         with self.assertRaises(AssertionError):
-            openstack_utils.resource_removed(resource_mock, 'e01df65a')
+            openstack_utils._resource_removed(resource_mock, 'e01df65a')
+
+    def test_resource_removed(self):
+        self.patch_object(openstack_utils, "_resource_removed")
+        self._resource_removed.return_value = True
+        openstack_utils.resource_removed('resource', 'e01df65a')
+        self._resource_removed.assert_called_once_with(
+            'resource',
+            'e01df65a',
+            'resource')
+
+    def test_resource_removed_custom_retry(self):
+        self.patch_object(openstack_utils, "_resource_removed")
+
+        def _retryer(f, arg1, arg2, arg3):
+            f(arg1, arg2, arg3)
+        self.patch_object(
+            openstack_utils.tenacity,
+            "Retrying",
+            return_value=_retryer)
+        saa_mock = mock.MagicMock()
+        self.patch_object(
+            openstack_utils.tenacity,
+            "stop_after_attempt",
+            return_value=saa_mock)
+        we_mock = mock.MagicMock()
+        self.patch_object(
+            openstack_utils.tenacity,
+            "wait_exponential",
+            return_value=we_mock)
+        self._resource_removed.return_value = True
+        openstack_utils.resource_removed(
+            'resource',
+            'e01df65a',
+            wait_exponential_multiplier=2,
+            wait_iteration_max_time=20,
+            stop_after_attempt=2)
+        self._resource_removed.assert_called_once_with(
+            'resource',
+            'e01df65a',
+            'resource')
+        self.Retrying.assert_called_once_with(
+            wait=we_mock,
+            reraise=True,
+            stop=saa_mock)
 
     def test_delete_resource(self):
         resource_mock = mock.MagicMock()

zaza/openstack/charm_tests/test_utils.py

@@ -217,7 +217,8 @@ class OpenStackBaseTest(unittest.TestCase):
                 services,
                 model_name=self.model_name)
 
-        logging.debug('Waiting for updates to propagate to '.format(config_file))
+        logging.debug(
+            'Waiting for updates to propagate to '.format(config_file))
         model.block_until_oslo_config_entries_match(
             self.application_name,
             config_file,

zaza/openstack/utilities/openstack.py

@@ -1593,9 +1593,7 @@ def resource_reaches_status(resource,
         msg)
 
 
-@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
-                reraise=True, stop=tenacity.stop_after_attempt(2))
-def resource_removed(resource, resource_id, msg="resource"):
+def _resource_removed(resource, resource_id, msg="resource"):
     """Wait for an openstack resource to no longer be present.
 
     :param resource: pointer to os resource type, ex: heat_client.stacks
@@ -1603,7 +1601,7 @@ def resource_removed(resource, resource_id, msg="resource"):
     :param resource_id: unique id for the openstack resource
     :type resource_id: str
     :param msg: text to identify purpose in logging
-    :type msy: str
+    :type msg: str
     :raises: AssertionError
     """
     matching = [r for r in resource.list() if r.id == resource_id]
@@ -1611,6 +1609,44 @@ def resource_removed(resource, resource_id, msg="resource"):
     assert len(matching) == 0, "Resource {} still present".format(resource_id)
 
 
+def resource_removed(resource,
+                     resource_id,
+                     msg='resource',
+                     wait_exponential_multiplier=1,
+                     wait_iteration_max_time=60,
+                     stop_after_attempt=8):
+    """Wait for an openstack resource to no longer be present.
+
+    :param resource: pointer to os resource type, ex: heat_client.stacks
+    :type resource: str
+    :param resource_id: unique id for the openstack resource
+    :type resource_id: str
+    :param msg: text to identify purpose in logging
+    :type msg: str
+    :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier
+                                        seconds between each retry
+    :type wait_exponential_multiplier: int
+    :param wait_iteration_max_time: Wait a max of wait_iteration_max_time
+                                    between retries.
+    :type wait_iteration_max_time: int
+    :param stop_after_attempt: Stop after stop_after_attempt retires.
+    :type stop_after_attempt: int
+    :raises: AssertionError
+    """
+    retryer = tenacity.Retrying(
+        wait=tenacity.wait_exponential(
+            multiplier=wait_exponential_multiplier,
+            max=wait_iteration_max_time),
+        reraise=True,
+        stop=tenacity.stop_after_attempt(stop_after_attempt))
+    print(retryer)
+    retryer(
+        _resource_removed,
+        resource,
+        resource_id,
+        msg)
+
+
 def delete_resource(resource, resource_id, msg="resource"):
     """Delete an openstack resource.