From d90da772d251d5a7204a64d403b3438f2b504906 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 27 Mar 2019 12:02:47 +0000 Subject: [PATCH] Expose resource_reaches_status' retry options (#206) * Expose resource_reaches_status' retry options resource_reaches_status is used in many places to monitor many different kinds of OpenStack resource. It is reasonable to expect a flavor to be created in a few seconds where as an image based instance may take minutes to become active. With that in mind this change exposes the retry options used by tenacity allowing the caller to set reasonable expectations for the resource state change. In addition the image based instance creation call is updated to retry for longer as this has been timing out. * Remove swap files --- .../test_zaza_utilities_openstack.py | 46 +++++++++++----- zaza/configure/guest.py | 3 +- zaza/utilities/openstack.py | 55 +++++++++++++++++-- 3 files changed, 85 insertions(+), 19 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 0b02a34..6d24313 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -352,47 +352,67 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.urlretrieve.assert_called_once_with( 'http://cirros/c.img', '/tmp/c1.img') - def test_resource_reaches_status(self): + def test__resource_reaches_status(self): resource_mock = mock.MagicMock() resource_mock.get.return_value = mock.MagicMock(status='available') - openstack_utils.resource_reaches_status(resource_mock, 'e01df65a') + openstack_utils._resource_reaches_status(resource_mock, 'e01df65a') - def test_resource_reaches_status_fail(self): - openstack_utils.resource_reaches_status.retry.wait = \ - tenacity.wait_none() + def test__resource_reaches_status_fail(self): resource_mock = mock.MagicMock() resource_mock.get.return_value = mock.MagicMock(status='unavailable') with self.assertRaises(AssertionError): - openstack_utils.resource_reaches_status( + openstack_utils._resource_reaches_status( resource_mock, 'e01df65a') - def test_resource_reaches_status_bespoke(self): + def test__resource_reaches_status_bespoke(self): resource_mock = mock.MagicMock() resource_mock.get.return_value = mock.MagicMock(status='readyish') - openstack_utils.resource_reaches_status( + openstack_utils._resource_reaches_status( resource_mock, 'e01df65a', 'readyish') - def test_resource_reaches_status_bespoke_fail(self): - openstack_utils.resource_reaches_status.retry.wait = \ - tenacity.wait_none() + def test__resource_reaches_status_bespoke_fail(self): resource_mock = mock.MagicMock() resource_mock.get.return_value = mock.MagicMock(status='available') with self.assertRaises(AssertionError): - openstack_utils.resource_reaches_status( + openstack_utils._resource_reaches_status( resource_mock, 'e01df65a', 'readyish') + def test_resource_reaches_status(self): + self.patch_object(openstack_utils, "_resource_reaches_status") + self._resource_reaches_status.return_value = True + openstack_utils._resource_reaches_status('resource', 'e01df65a') + self._resource_reaches_status.assert_called_once_with( + 'resource', + 'e01df65a') + + def test_resource_reaches_status_custom_retry(self): + self.patch_object(openstack_utils, "_resource_reaches_status") + self._resource_reaches_status.return_value = True + openstack_utils._resource_reaches_status( + 'resource', + 'e01df65a', + wait_exponential_multiplier=2, + wait_iteration_max_time=20, + stop_after_attempt=2) + self._resource_reaches_status.assert_called_once_with( + 'resource', + 'e01df65a', + stop_after_attempt=2, + wait_exponential_multiplier=2, + wait_iteration_max_time=20) + def test_resource_removed(self): resource_mock = mock.MagicMock() resource_mock.list.return_value = [mock.MagicMock(id='ba8204b0')] openstack_utils.resource_removed(resource_mock, 'e01df65a') def test_resource_removed_fail(self): - openstack_utils.resource_reaches_status.retry.wait = \ + openstack_utils.resource_removed.retry.wait = \ tenacity.wait_none() resource_mock = mock.MagicMock() resource_mock.list.return_value = [mock.MagicMock(id='e01df65a')] diff --git a/zaza/configure/guest.py b/zaza/configure/guest.py index 02b5a52..ad0cd18 100644 --- a/zaza/configure/guest.py +++ b/zaza/configure/guest.py @@ -109,7 +109,8 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, openstack_utils.resource_reaches_status( nova_client.servers, instance.id, - expected_status='ACTIVE') + expected_status='ACTIVE', + stop_after_attempt=16) logging.info('Checking cloud init is complete') openstack_utils.cloud_init_complete( diff --git a/zaza/utilities/openstack.py b/zaza/utilities/openstack.py index a5e8487..6dc48e1 100644 --- a/zaza/utilities/openstack.py +++ b/zaza/utilities/openstack.py @@ -1519,11 +1519,9 @@ def download_image(image_url, target_file): urllib.request.urlretrieve(image_url, target_file) -@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), - reraise=True, stop=tenacity.stop_after_attempt(8)) -def resource_reaches_status(resource, resource_id, - expected_status='available', - msg='resource'): +def _resource_reaches_status(resource, resource_id, + expected_status='available', + msg='resource'): """Wait for an openstack resources status to reach an expected status. Wait for an openstack resources status to reach an expected status @@ -1548,6 +1546,53 @@ def resource_reaches_status(resource, resource_id, expected_status,)) +def resource_reaches_status(resource, + resource_id, + expected_status='available', + msg='resource', + wait_exponential_multiplier=1, + wait_iteration_max_time=60, + stop_after_attempt=8, + ): + """Wait for an openstack resources status to reach an expected status. + + Wait for an openstack resources status to reach an expected status + within a specified time. Useful to confirm that nova instances, cinder + vols, snapshots, glance images, heat stacks and other resources + eventually reach the expected status. + + :param resource: pointer to os resource type, ex: heat_client.stacks + :type resource: str + :param resource_id: unique id for the openstack resource + :type resource_id: str + :param expected_status: status to expect resource to reach + :type expected_status: str + :param msg: text to identify purpose in logging + :type msg: str + :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier + seconds between each retry + :type wait_exponential_multiplier: int + :param wait_iteration_max_time: Wait a max of wait_iteration_max_time + between retries. + :type wait_iteration_max_time: int + :param stop_after_attempt: Stop after stop_after_attempt retires. + :type stop_after_attempt: int + :raises: AssertionError + """ + retryer = tenacity.Retrying( + wait=tenacity.wait_exponential( + multiplier=wait_exponential_multiplier, + max=wait_iteration_max_time), + reraise=True, + stop=tenacity.stop_after_attempt(stop_after_attempt)) + retryer( + _resource_reaches_status, + resource, + resource_id, + expected_status, + msg) + + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(2)) def resource_removed(resource, resource_id, msg="resource"):