Port Rmq charm func test removing a unit from a Rmq cluster

This function was previously called test_901_remove_unit, but had to
be renamed (moved to the end of the func tests); The way in which unit
removal is now performed (by running the "stop" hook) puts the the
removed unit in a "waiting" state -- which consequently causes
wait_for_cluster() (e.g. used in 910) to fail (timeout).

zaza/openstack/charm_tests/rabbitmq_server/tests.py

@@ -363,3 +363,44 @@ class RmqTests(test_utils.OpenStackBaseTest):
         assert queue_data['messages'] == 0, 'Found unexpected message count.'
 
         logging.debug('OK')
+
+    def test_921_remove_unit(self):
+        """Test if unit cleans up when removed from Rmq cluster.
+
+        Test if a unit correctly cleans up by removing itself from the
+        RabbitMQ cluster on removal
+
+        """
+        logging.debug('Checking that units correctly clean up after '
+                      'themselves on unit removal...')
+        config = {'min-cluster-size': '2'}
+        zaza.model.set_application_config('rabbitmq-server', config)
+        rmq_utils.wait_for_cluster()
+
+        units = zaza.model.get_units(self.application_name)
+        removed_unit = units[-1]
+        left_units = units[:-1]
+
+        zaza.model.run_on_unit(removed_unit.entity_id, 'hooks/stop')
+        zaza.model.block_until_unit_wl_status(removed_unit.entity_id,
+                                              "waiting")
+
+        unit_host_names = generic_utils.get_unit_hostnames(left_units)
+        unit_node_names = []
+        for unit in unit_host_names:
+            unit_node_names.append('rabbit@{}'.format(unit_host_names[unit]))
+        errors = []
+
+        for u in left_units:
+            e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names)
+            if e:
+                # NOTE: cluster status may not have been updated yet so wait a
+                # little and try one more time. Need to find a better way to do
+                # this.
+                time.sleep(10)
+                e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names)
+                if e:
+                    errors.append(e)
+
+        self.assertFalse(errors)
+        logging.debug('OK')

zaza/openstack/charm_tests/rabbitmq_server/utils.py

@@ -439,3 +439,26 @@ def get_amqp_message_by_unit(unit, queue="test",
     else:
         msg = 'No message retrieved.'
         raise Exception(msg)
+
+
+def check_unit_cluster_nodes(unit, unit_node_names):
+    """Check if unit exists in list of Rmq cluster node names."""
+    unit_name = unit.entity_id
+    nodes = []
+    errors = []
+    str_stat = get_cluster_status(unit)
+    # make the interesting part of rabbitmqctl cluster_status output
+    # json-parseable.
+    if 'nodes,[{disc,' in str_stat:
+        pos_start = str_stat.find('nodes,[{disc,') + 13
+        pos_end = str_stat.find(']}]},', pos_start) + 1
+        str_nodes = str_stat[pos_start:pos_end].replace("'", '"')
+        nodes = json.loads(str_nodes)
+    for node in nodes:
+        if node not in unit_node_names:
+            errors.append('Cluster registration check failed on {}: '
+                          '{} should not be registered with RabbitMQ '
+                          'after unit removal.\n'
+                          ''.format(unit_name, node))
+
+    return errors