Merge pull request #279 from ajkavanagh/tenacity-for-instance-ping

Add retries to instance pinging

zaza/openstack/configure/guest.py

@@ -22,6 +22,14 @@ import time
 
 import zaza.openstack.utilities.openstack as openstack_utils
 import zaza.openstack.charm_tests.nova.utils as nova_utils
+import zaza.openstack.utilities.exceptions as openstack_exceptions
+
+from tenacity import (
+    RetryError,
+    Retrying,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 boot_tests = {
     'cirros': {
@@ -134,12 +142,20 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
         port=port)['floating_ip_address']
     logging.info('Assigned floating IP {} to {}'.format(ip, vm_name))
     try:
-        openstack_utils.ping_response(ip)
-    except subprocess.CalledProcessError as e:
-        logging.error('Pinging {} failed with {}'.format(ip, e.returncode))
-        logging.error('stdout: {}'.format(e.stdout))
-        logging.error('stderr: {}'.format(e.stderr))
-        raise
+        for attempt in Retrying(
+                stop=stop_after_attempt(8),
+                wait=wait_exponential(multiplier=1, min=2, max=60)):
+            with attempt:
+                try:
+                    openstack_utils.ping_response(ip)
+                except subprocess.CalledProcessError as e:
+                    logging.error('Pinging {} failed with {}'
+                                  .format(ip, e.returncode))
+                    logging.error('stdout: {}'.format(e.stdout))
+                    logging.error('stderr: {}'.format(e.stderr))
+                    raise
+    except RetryError:
+        raise openstack_exceptions.NovaGuestNoPingResponse()
 
     # Check ssh'ing to instance.
     logging.info('Testing ssh access.')

zaza/openstack/utilities/exceptions.py

@@ -180,6 +180,12 @@ class NovaGuestRestartFailed(Exception):
     pass
 
 
+class NovaGuestNoPingResponse(Exception):
+    """Nova guest failed to respond to pings."""
+
+    pass
+
+
 class PolicydError(Exception):
     """Policyd override failed."""