From 648ac9262bfa3a4e40aeeb93c267619f09c1d0b1 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 23 May 2019 10:16:03 +0100 Subject: [PATCH] Expose resource_removeds retry options (#11) resource_removed is used in many places to monitor removal of different kinds of OpenStack resource. It is reasonable to expect a flavor to be removed in a few seconds where as an image based instance may take minutes to be removed. With that in mind this change exposes the retry options used by tenacity allowing the caller to set reasonable expectations for the resource removal. --- .../test_zaza_utilities_openstack.py | 54 ++++++++++++++++--- zaza/openstack/charm_tests/test_utils.py | 3 +- zaza/openstack/utilities/openstack.py | 44 +++++++++++++-- 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index d40ccf8..1ced14b 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -406,18 +406,60 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): wait_exponential_multiplier=2, wait_iteration_max_time=20) - def test_resource_removed(self): + def test__resource_removed(self): resource_mock = mock.MagicMock() resource_mock.list.return_value = [mock.MagicMock(id='ba8204b0')] - openstack_utils.resource_removed(resource_mock, 'e01df65a') + openstack_utils._resource_removed(resource_mock, 'e01df65a') - def test_resource_removed_fail(self): - openstack_utils.resource_removed.retry.wait = \ - tenacity.wait_none() + def test__resource_removed_fail(self): resource_mock = mock.MagicMock() resource_mock.list.return_value = [mock.MagicMock(id='e01df65a')] with self.assertRaises(AssertionError): - openstack_utils.resource_removed(resource_mock, 'e01df65a') + openstack_utils._resource_removed(resource_mock, 'e01df65a') + + def test_resource_removed(self): + self.patch_object(openstack_utils, "_resource_removed") + self._resource_removed.return_value = True + openstack_utils.resource_removed('resource', 'e01df65a') + self._resource_removed.assert_called_once_with( + 'resource', + 'e01df65a', + 'resource') + + def test_resource_removed_custom_retry(self): + self.patch_object(openstack_utils, "_resource_removed") + + def _retryer(f, arg1, arg2, arg3): + f(arg1, arg2, arg3) + self.patch_object( + openstack_utils.tenacity, + "Retrying", + return_value=_retryer) + saa_mock = mock.MagicMock() + self.patch_object( + openstack_utils.tenacity, + "stop_after_attempt", + return_value=saa_mock) + we_mock = mock.MagicMock() + self.patch_object( + openstack_utils.tenacity, + "wait_exponential", + return_value=we_mock) + self._resource_removed.return_value = True + openstack_utils.resource_removed( + 'resource', + 'e01df65a', + wait_exponential_multiplier=2, + wait_iteration_max_time=20, + stop_after_attempt=2) + self._resource_removed.assert_called_once_with( + 'resource', + 'e01df65a', + 'resource') + self.Retrying.assert_called_once_with( + wait=we_mock, + reraise=True, + stop=saa_mock) def test_delete_resource(self): resource_mock = mock.MagicMock() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 376eca0..b9eba4f 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -217,7 +217,8 @@ class OpenStackBaseTest(unittest.TestCase): services, model_name=self.model_name) - logging.debug('Waiting for updates to propagate to '.format(config_file)) + logging.debug( + 'Waiting for updates to propagate to '.format(config_file)) model.block_until_oslo_config_entries_match( self.application_name, config_file, diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 10da3fb..f88fef2 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1593,9 +1593,7 @@ def resource_reaches_status(resource, msg) -@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), - reraise=True, stop=tenacity.stop_after_attempt(2)) -def resource_removed(resource, resource_id, msg="resource"): +def _resource_removed(resource, resource_id, msg="resource"): """Wait for an openstack resource to no longer be present. :param resource: pointer to os resource type, ex: heat_client.stacks @@ -1603,7 +1601,7 @@ def resource_removed(resource, resource_id, msg="resource"): :param resource_id: unique id for the openstack resource :type resource_id: str :param msg: text to identify purpose in logging - :type msy: str + :type msg: str :raises: AssertionError """ matching = [r for r in resource.list() if r.id == resource_id] @@ -1611,6 +1609,44 @@ def resource_removed(resource, resource_id, msg="resource"): assert len(matching) == 0, "Resource {} still present".format(resource_id) +def resource_removed(resource, + resource_id, + msg='resource', + wait_exponential_multiplier=1, + wait_iteration_max_time=60, + stop_after_attempt=8): + """Wait for an openstack resource to no longer be present. + + :param resource: pointer to os resource type, ex: heat_client.stacks + :type resource: str + :param resource_id: unique id for the openstack resource + :type resource_id: str + :param msg: text to identify purpose in logging + :type msg: str + :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier + seconds between each retry + :type wait_exponential_multiplier: int + :param wait_iteration_max_time: Wait a max of wait_iteration_max_time + between retries. + :type wait_iteration_max_time: int + :param stop_after_attempt: Stop after stop_after_attempt retires. + :type stop_after_attempt: int + :raises: AssertionError + """ + retryer = tenacity.Retrying( + wait=tenacity.wait_exponential( + multiplier=wait_exponential_multiplier, + max=wait_iteration_max_time), + reraise=True, + stop=tenacity.stop_after_attempt(stop_after_attempt)) + print(retryer) + retryer( + _resource_removed, + resource, + resource_id, + msg) + + def delete_resource(resource, resource_id, msg="resource"): """Delete an openstack resource.