diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 0b02a34..6d24313 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -352,47 +352,67 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.urlretrieve.assert_called_once_with( 'http://cirros/c.img', '/tmp/c1.img') - def test_resource_reaches_status(self): + def test__resource_reaches_status(self): resource_mock = mock.MagicMock() resource_mock.get.return_value = mock.MagicMock(status='available') - openstack_utils.resource_reaches_status(resource_mock, 'e01df65a') + openstack_utils._resource_reaches_status(resource_mock, 'e01df65a') - def test_resource_reaches_status_fail(self): - openstack_utils.resource_reaches_status.retry.wait = \ - tenacity.wait_none() + def test__resource_reaches_status_fail(self): resource_mock = mock.MagicMock() resource_mock.get.return_value = mock.MagicMock(status='unavailable') with self.assertRaises(AssertionError): - openstack_utils.resource_reaches_status( + openstack_utils._resource_reaches_status( resource_mock, 'e01df65a') - def test_resource_reaches_status_bespoke(self): + def test__resource_reaches_status_bespoke(self): resource_mock = mock.MagicMock() resource_mock.get.return_value = mock.MagicMock(status='readyish') - openstack_utils.resource_reaches_status( + openstack_utils._resource_reaches_status( resource_mock, 'e01df65a', 'readyish') - def test_resource_reaches_status_bespoke_fail(self): - openstack_utils.resource_reaches_status.retry.wait = \ - tenacity.wait_none() + def test__resource_reaches_status_bespoke_fail(self): resource_mock = mock.MagicMock() resource_mock.get.return_value = mock.MagicMock(status='available') with self.assertRaises(AssertionError): - openstack_utils.resource_reaches_status( + openstack_utils._resource_reaches_status( resource_mock, 'e01df65a', 'readyish') + def test_resource_reaches_status(self): + self.patch_object(openstack_utils, "_resource_reaches_status") + self._resource_reaches_status.return_value = True + openstack_utils._resource_reaches_status('resource', 'e01df65a') + self._resource_reaches_status.assert_called_once_with( + 'resource', + 'e01df65a') + + def test_resource_reaches_status_custom_retry(self): + self.patch_object(openstack_utils, "_resource_reaches_status") + self._resource_reaches_status.return_value = True + openstack_utils._resource_reaches_status( + 'resource', + 'e01df65a', + wait_exponential_multiplier=2, + wait_iteration_max_time=20, + stop_after_attempt=2) + self._resource_reaches_status.assert_called_once_with( + 'resource', + 'e01df65a', + stop_after_attempt=2, + wait_exponential_multiplier=2, + wait_iteration_max_time=20) + def test_resource_removed(self): resource_mock = mock.MagicMock() resource_mock.list.return_value = [mock.MagicMock(id='ba8204b0')] openstack_utils.resource_removed(resource_mock, 'e01df65a') def test_resource_removed_fail(self): - openstack_utils.resource_reaches_status.retry.wait = \ + openstack_utils.resource_removed.retry.wait = \ tenacity.wait_none() resource_mock = mock.MagicMock() resource_mock.list.return_value = [mock.MagicMock(id='e01df65a')] diff --git a/zaza/configure/guest.py b/zaza/configure/guest.py index 02b5a52..ad0cd18 100644 --- a/zaza/configure/guest.py +++ b/zaza/configure/guest.py @@ -109,7 +109,8 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, openstack_utils.resource_reaches_status( nova_client.servers, instance.id, - expected_status='ACTIVE') + expected_status='ACTIVE', + stop_after_attempt=16) logging.info('Checking cloud init is complete') openstack_utils.cloud_init_complete( diff --git a/zaza/utilities/openstack.py b/zaza/utilities/openstack.py index a5e8487..6dc48e1 100644 --- a/zaza/utilities/openstack.py +++ b/zaza/utilities/openstack.py @@ -1519,11 +1519,9 @@ def download_image(image_url, target_file): urllib.request.urlretrieve(image_url, target_file) -@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), - reraise=True, stop=tenacity.stop_after_attempt(8)) -def resource_reaches_status(resource, resource_id, - expected_status='available', - msg='resource'): +def _resource_reaches_status(resource, resource_id, + expected_status='available', + msg='resource'): """Wait for an openstack resources status to reach an expected status. Wait for an openstack resources status to reach an expected status @@ -1548,6 +1546,53 @@ def resource_reaches_status(resource, resource_id, expected_status,)) +def resource_reaches_status(resource, + resource_id, + expected_status='available', + msg='resource', + wait_exponential_multiplier=1, + wait_iteration_max_time=60, + stop_after_attempt=8, + ): + """Wait for an openstack resources status to reach an expected status. + + Wait for an openstack resources status to reach an expected status + within a specified time. Useful to confirm that nova instances, cinder + vols, snapshots, glance images, heat stacks and other resources + eventually reach the expected status. + + :param resource: pointer to os resource type, ex: heat_client.stacks + :type resource: str + :param resource_id: unique id for the openstack resource + :type resource_id: str + :param expected_status: status to expect resource to reach + :type expected_status: str + :param msg: text to identify purpose in logging + :type msg: str + :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier + seconds between each retry + :type wait_exponential_multiplier: int + :param wait_iteration_max_time: Wait a max of wait_iteration_max_time + between retries. + :type wait_iteration_max_time: int + :param stop_after_attempt: Stop after stop_after_attempt retires. + :type stop_after_attempt: int + :raises: AssertionError + """ + retryer = tenacity.Retrying( + wait=tenacity.wait_exponential( + multiplier=wait_exponential_multiplier, + max=wait_iteration_max_time), + reraise=True, + stop=tenacity.stop_after_attempt(stop_after_attempt)) + retryer( + _resource_reaches_status, + resource, + resource_id, + expected_status, + msg) + + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(2)) def resource_removed(resource, resource_id, msg="resource"):