Don't sleep before attempting get AMQP msg. Retry w/ tenacity

zaza/openstack/charm_tests/rabbitmq_server/tests.py

@@ -20,6 +20,7 @@ import time
 import uuid
 
 import juju
+import tenacity
 import zaza.model
 import zaza.openstack.charm_tests.test_utils as test_utils
 import zaza.openstack.utilities.generic as generic_utils
@@ -32,6 +33,7 @@ from zaza.openstack.utilities.generic import (
 )
 
 from . import utils as rmq_utils
+from .utils import RmqNoMessageException
 
 
 class RmqTests(test_utils.OpenStackBaseTest):
@@ -49,6 +51,15 @@ class RmqTests(test_utils.OpenStackBaseTest):
         """
         return '[{}-{}]'.format(uuid.uuid4(), time.time())
 
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(RmqNoMessageException),
+        wait=tenacity.wait_fixed(10),
+        stop=tenacity.stop_after_attempt(2))
+    def _retry_get_amqp_message(self, check_unit, ssl=None, port=None):
+        return rmq_utils.get_amqp_message_by_unit(check_unit,
+                                                  ssl=ssl,
+                                                  port=port)
+
     def _test_rmq_amqp_messages_all_units(self, units,
                                           ssl=False, port=None):
         """Reusable test to send/check amqp messages to every listed rmq unit.
@@ -97,18 +108,15 @@ class RmqTests(test_utils.OpenStackBaseTest):
                                                        amqp_msg, ssl=ssl,
                                                        port=port)
 
-                # Wait a bit before checking for message
-                time.sleep(10)
-
                 # Get amqp message
                 logging.debug('Get message from:   {} '
                               '({} {})'.format(check_unit_host,
                                                check_unit_name,
                                                check_unit_host_name))
 
-                amqp_msg_rcvd = rmq_utils.get_amqp_message_by_unit(check_unit,
-                                                                   ssl=ssl,
-                                                                   port=port)
+                amqp_msg_rcvd = self._retry_get_amqp_message(check_unit,
+                                                             ssl=ssl,
+                                                             port=port)
 
                 # Validate amqp message content
                 if amqp_msg == amqp_msg_rcvd:

zaza/openstack/charm_tests/rabbitmq_server/utils.py

@@ -25,6 +25,12 @@ import ssl as libssl
 import zaza.openstack.utilities.generic as generic_utils
 
 
+class RmqNoMessageException(Exception):
+    """Message retrieval from Rmq resulted in no message."""
+
+    pass
+
+
 def wait_for_cluster(model_name=None, timeout=1200):
     """Wait for Rmq cluster status to show cluster readiness.
 
@@ -438,7 +444,7 @@ def get_amqp_message_by_unit(unit, queue="test",
         return body
     else:
         msg = 'No message retrieved.'
-        raise Exception(msg)
+        raise RmqNoMessageException(msg)
 
 
 def check_unit_cluster_nodes(unit, unit_node_names):