Add test confirming that all Rmq units are part of cluster

zaza/openstack/charm_tests/rabbitmq_server/tests.py

@@ -12,8 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+import zaza.model
 import zaza.openstack.charm_tests.test_utils as test_utils
 
+from . import utils as rmq_utils
+
 
 class RmqTests(test_utils.OpenStackBaseTest):
     """Zaza tests on a basic rabbitmq cluster deployment. Verify
@@ -24,3 +28,16 @@ class RmqTests(test_utils.OpenStackBaseTest):
         """Run class setup for running tests."""
         super(RmqTests, cls).setUpClass()
 
+    def test_400_rmq_cluster_running_nodes(self):
+        """Verify that cluster status from each rmq juju unit shows
+        every cluster node as a running member in that cluster."""
+        logging.debug('Checking that all units are in cluster_status '
+                      'running nodes...')
+
+        units = zaza.model.get_units(self.application_name)
+
+        ret = rmq_utils.validate_cluster_running_nodes(units)
+        self.assertIsNone(ret)
+
+        logging.info('OK\n')
+

zaza/openstack/charm_tests/rabbitmq_server/utils.py

@@ -19,6 +19,8 @@ import pika
 import zaza.model
 
 import ssl as libssl
+import zaza.openstack.utilities.generic as generic_utils
+
 
 def wait_for_cluster(model_name=None, timeout=1200):
     """Wait for rmq units extended status to show cluster readiness,
@@ -68,6 +70,36 @@ def get_cluster_running_nodes(unit):
         return []
 
 
+def validate_cluster_running_nodes(units):
+    """Check that all rmq unit hostnames are represented in the
+    cluster_status output of all units.
+    :param host_names: dict of juju unit names to host names
+    :param units: list of unit pointers (all rmq units)
+    :returns: None if successful, otherwise return error message
+    """
+    host_names = generic_utils.get_unit_hostnames(units)
+    errors = []
+
+    # Query every unit for cluster_status running nodes
+    for query_unit in units:
+        query_unit_name = query_unit.entity_id
+        running_nodes = get_cluster_running_nodes(query_unit)
+
+        # Confirm that every unit is represented in the queried unit's
+        # cluster_status running nodes output.
+        for validate_unit in units:
+            val_host_name = host_names[validate_unit.entity_id]
+            val_node_name = 'rabbit@{}'.format(val_host_name)
+
+            if val_node_name not in running_nodes:
+                errors.append('Cluster member check failed on {}: {} not '
+                              'in {}\n'.format(query_unit_name,
+                                               val_node_name,
+                                               running_nodes))
+    if errors:
+        return ''.join(errors)
+
+
 def connect_amqp_by_unit(unit, ssl=False,
                          port=None, fatal=True,
                          username="testuser1", password="changeme"):