diff --git a/zaza/openstack/configure/guest.py b/zaza/openstack/configure/guest.py index bd70bca..c2e20de 100644 --- a/zaza/openstack/configure/guest.py +++ b/zaza/openstack/configure/guest.py @@ -22,6 +22,14 @@ import time import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.nova.utils as nova_utils +import zaza.openstack.utilities.exceptions as openstack_exceptions + +from tenacity import ( + RetryError, + Retrying, + stop_after_attempt, + wait_exponential, +) boot_tests = { 'cirros': { @@ -134,12 +142,20 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, port=port)['floating_ip_address'] logging.info('Assigned floating IP {} to {}'.format(ip, vm_name)) try: - openstack_utils.ping_response(ip) - except subprocess.CalledProcessError as e: - logging.error('Pinging {} failed with {}'.format(ip, e.returncode)) - logging.error('stdout: {}'.format(e.stdout)) - logging.error('stderr: {}'.format(e.stderr)) - raise + for attempt in Retrying( + stop=stop_after_attempt(8), + wait=wait_exponential(multiplier=1, min=2, max=60)): + with attempt: + try: + openstack_utils.ping_response(ip) + except subprocess.CalledProcessError as e: + logging.error('Pinging {} failed with {}' + .format(ip, e.returncode)) + logging.error('stdout: {}'.format(e.stdout)) + logging.error('stderr: {}'.format(e.stderr)) + raise + except RetryError: + raise openstack_exceptions.NovaGuestNoPingResponse() # Check ssh'ing to instance. logging.info('Testing ssh access.') diff --git a/zaza/openstack/utilities/exceptions.py b/zaza/openstack/utilities/exceptions.py index e9e5d23..ab4e258 100644 --- a/zaza/openstack/utilities/exceptions.py +++ b/zaza/openstack/utilities/exceptions.py @@ -180,6 +180,12 @@ class NovaGuestRestartFailed(Exception): pass +class NovaGuestNoPingResponse(Exception): + """Nova guest failed to respond to pings.""" + + pass + + class PolicydError(Exception): """Policyd override failed."""