Expose resource_reaches_status' retry options (#206)

* Expose resource_reaches_status' retry options

resource_reaches_status is used in many places to monitor many
different kinds of OpenStack resource. It is reasonable to expect a
flavor to be created in a few seconds where as an image based instance
may take minutes to become active. With that in mind this change
exposes the retry options used by tenacity allowing the caller to
set reasonable expectations for the resource state change.

In addition the image based instance creation call is updated
to retry for longer as this has been timing out.

* Remove swap files

unit_tests/utilities/test_zaza_utilities_openstack.py

@@ -352,47 +352,67 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.urlretrieve.assert_called_once_with(
             'http://cirros/c.img', '/tmp/c1.img')
 
-    def test_resource_reaches_status(self):
+    def test__resource_reaches_status(self):
         resource_mock = mock.MagicMock()
         resource_mock.get.return_value = mock.MagicMock(status='available')
-        openstack_utils.resource_reaches_status(resource_mock, 'e01df65a')
+        openstack_utils._resource_reaches_status(resource_mock, 'e01df65a')
 
-    def test_resource_reaches_status_fail(self):
-        openstack_utils.resource_reaches_status.retry.wait = \
-            tenacity.wait_none()
+    def test__resource_reaches_status_fail(self):
         resource_mock = mock.MagicMock()
         resource_mock.get.return_value = mock.MagicMock(status='unavailable')
         with self.assertRaises(AssertionError):
-            openstack_utils.resource_reaches_status(
+            openstack_utils._resource_reaches_status(
                 resource_mock,
                 'e01df65a')
 
-    def test_resource_reaches_status_bespoke(self):
+    def test__resource_reaches_status_bespoke(self):
         resource_mock = mock.MagicMock()
         resource_mock.get.return_value = mock.MagicMock(status='readyish')
-        openstack_utils.resource_reaches_status(
+        openstack_utils._resource_reaches_status(
             resource_mock,
             'e01df65a',
             'readyish')
 
-    def test_resource_reaches_status_bespoke_fail(self):
-        openstack_utils.resource_reaches_status.retry.wait = \
-            tenacity.wait_none()
+    def test__resource_reaches_status_bespoke_fail(self):
         resource_mock = mock.MagicMock()
         resource_mock.get.return_value = mock.MagicMock(status='available')
         with self.assertRaises(AssertionError):
-            openstack_utils.resource_reaches_status(
+            openstack_utils._resource_reaches_status(
                 resource_mock,
                 'e01df65a',
                 'readyish')
 
+    def test_resource_reaches_status(self):
+        self.patch_object(openstack_utils, "_resource_reaches_status")
+        self._resource_reaches_status.return_value = True
+        openstack_utils._resource_reaches_status('resource', 'e01df65a')
+        self._resource_reaches_status.assert_called_once_with(
+            'resource',
+            'e01df65a')
+
+    def test_resource_reaches_status_custom_retry(self):
+        self.patch_object(openstack_utils, "_resource_reaches_status")
+        self._resource_reaches_status.return_value = True
+        openstack_utils._resource_reaches_status(
+            'resource',
+            'e01df65a',
+            wait_exponential_multiplier=2,
+            wait_iteration_max_time=20,
+            stop_after_attempt=2)
+        self._resource_reaches_status.assert_called_once_with(
+            'resource',
+            'e01df65a',
+            stop_after_attempt=2,
+            wait_exponential_multiplier=2,
+            wait_iteration_max_time=20)
+
     def test_resource_removed(self):
         resource_mock = mock.MagicMock()
         resource_mock.list.return_value = [mock.MagicMock(id='ba8204b0')]
         openstack_utils.resource_removed(resource_mock, 'e01df65a')
 
     def test_resource_removed_fail(self):
-        openstack_utils.resource_reaches_status.retry.wait = \
+        openstack_utils.resource_removed.retry.wait = \
             tenacity.wait_none()
         resource_mock = mock.MagicMock()
         resource_mock.list.return_value = [mock.MagicMock(id='e01df65a')]

zaza/configure/guest.py

@@ -109,7 +109,8 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
     openstack_utils.resource_reaches_status(
         nova_client.servers,
         instance.id,
-        expected_status='ACTIVE')
+        expected_status='ACTIVE',
+        stop_after_attempt=16)
 
     logging.info('Checking cloud init is complete')
     openstack_utils.cloud_init_complete(

zaza/utilities/openstack.py

@@ -1519,11 +1519,9 @@ def download_image(image_url, target_file):
     urllib.request.urlretrieve(image_url, target_file)
 
 
-@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
-                reraise=True, stop=tenacity.stop_after_attempt(8))
-def resource_reaches_status(resource, resource_id,
-                            expected_status='available',
-                            msg='resource'):
+def _resource_reaches_status(resource, resource_id,
+                             expected_status='available',
+                             msg='resource'):
     """Wait for an openstack resources status to reach an expected status.
 
        Wait for an openstack resources status to reach an expected status
@@ -1548,6 +1546,53 @@ def resource_reaches_status(resource, resource_id,
                                                        expected_status,))
 
 
+def resource_reaches_status(resource,
+                            resource_id,
+                            expected_status='available',
+                            msg='resource',
+                            wait_exponential_multiplier=1,
+                            wait_iteration_max_time=60,
+                            stop_after_attempt=8,
+                            ):
+    """Wait for an openstack resources status to reach an expected status.
+
+       Wait for an openstack resources status to reach an expected status
+       within a specified time. Useful to confirm that nova instances, cinder
+       vols, snapshots, glance images, heat stacks and other resources
+       eventually reach the expected status.
+
+    :param resource: pointer to os resource type, ex: heat_client.stacks
+    :type resource: str
+    :param resource_id: unique id for the openstack resource
+    :type resource_id: str
+    :param expected_status: status to expect resource to reach
+    :type expected_status: str
+    :param msg: text to identify purpose in logging
+    :type msg: str
+    :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier
+                                        seconds between each retry
+    :type wait_exponential_multiplier: int
+    :param wait_iteration_max_time: Wait a max of wait_iteration_max_time
+                                    between retries.
+    :type wait_iteration_max_time: int
+    :param stop_after_attempt: Stop after stop_after_attempt retires.
+    :type stop_after_attempt: int
+    :raises: AssertionError
+    """
+    retryer = tenacity.Retrying(
+        wait=tenacity.wait_exponential(
+            multiplier=wait_exponential_multiplier,
+            max=wait_iteration_max_time),
+        reraise=True,
+        stop=tenacity.stop_after_attempt(stop_after_attempt))
+    retryer(
+        _resource_reaches_status,
+        resource,
+        resource_id,
+        expected_status,
+        msg)
+
+
 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
                 reraise=True, stop=tenacity.stop_after_attempt(2))
 def resource_removed(resource, resource_id, msg="resource"):