Add functional testing of remove-devices action

Refactored some swift-proxy ring query utilities for new tests

zaza/openstack/charm_tests/swift/tests.py

@@ -73,6 +73,8 @@ class SwiftProxyTests(test_utils.OpenStackBaseTest):
     """Tests specific to swift proxy."""
 
     TEST_SEARCH_TARGET = 'd0'
+    TEST_REMOVE_TARGET = 'd1'
+    TEST_EXPECTED_RING_HOSTS = 1
     TEST_WEIGHT_TARGET = 999
     TEST_WEIGHT_INITIAL = 100
 
@@ -96,8 +98,7 @@ class SwiftProxyTests(test_utils.OpenStackBaseTest):
         self.assertEqual(action.status, "completed")
 
     def test_904_set_weight_action_and_validate_rebalance(self):
-        """Set weight of device in object ring, rebalance, and
-        validate rings have rebalance and replicated"""
+        """Set weight of device in object ring."""
         logging.info('Running set-weight action on leader')
         action = zaza.model.run_action_on_leader(
             'swift-proxy',
@@ -108,24 +109,13 @@ class SwiftProxyTests(test_utils.OpenStackBaseTest):
         self.assertEqual(action.status, "completed")
 
         logging.info('Validating builder updated as expected')
-        cmd = ('swift-ring-builder /etc/swift/object.builder search {}'
-               ''.format(self.TEST_SEARCH_TARGET))
-        result = zaza.model.run_on_leader('swift-proxy', cmd)
-        self.assertTrue('Stdout' in result,
-                        'failed to retrieve swift-ring-builder weight')
+        result = swift_utils.search_builder('swift-proxy', 'object',
+                                            self.TEST_SEARCH_TARGET)
         # disk weight is the 9th field of the second line and is a float
-        disk_weight = result['Stdout'].split('\n')[1].split()[8]
-        disk_weight = int(disk_weight.split('.')[0])
+        disk_weight = int(result.split('\n')[1].split()[8].split('.')[0])
         self.assertEqual(disk_weight, self.TEST_WEIGHT_TARGET)
-
-        logging.info('Checking ring has rebalanced')
-        cmd = ('swift-ring-builder /etc/swift/object.builder | '
-               'grep "Ring file .* is"')
-        result = zaza.model.run_on_leader('swift-proxy', cmd)
-        self.assertTrue('Stdout' in result,
-                        'failed to retrieve object ring status')
-        expected = 'Ring file /etc/swift/object.ring.gz is up-to-date'
-        self.assertEqual(result['Stdout'].strip('\n'), expected)
+        self.assertTrue(swift_utils.is_proxy_ring_up_to_date('swift-proxy',
+                                                             'object'))
 
         logging.info('Running set-weight on leader to reset weight back')
         action = zaza.model.run_action_on_leader(
@@ -135,15 +125,30 @@ class SwiftProxyTests(test_utils.OpenStackBaseTest):
                            'search-value': self.TEST_SEARCH_TARGET,
                            'weight': self.TEST_WEIGHT_INITIAL})
         self.assertEqual(action.status, "completed")
+        self.assertTrue(
+            swift_utils.is_ring_synced('swift-proxy', 'object',
+                                       self.TEST_EXPECTED_RING_HOSTS))
 
-        logging.info('Checking ring md5sums on storage units against proxy')
-        zaza.model.block_until_all_units_idle()
-        cmd = 'swift-recon --md5 | grep -A1 "ring md5" | tail -1'
-        result = zaza.model.run_on_leader('swift-proxy', cmd)
-        self.assertTrue('Stdout' in result,
-                        'failed to retrieve swift-recon results')
-        expected = '1/1 hosts matched, 0 error[s] while checking hosts.'
-        self.assertEqual(result['Stdout'].strip('\n'), expected)
+    def test_905_remove_device_action_and_validate_rebalance(self):
+        """Remove device from object ring."""
+        logging.info('Running remove-devices action on leader')
+        action = zaza.model.run_action_on_leader(
+            'swift-proxy',
+            'remove-devices',
+            action_params={'ring': 'object',
+                           'search-value': self.TEST_REMOVE_TARGET})
+        self.assertEqual(action.status, "completed")
+
+        logging.info('Validating builder updated as expected')
+        result = swift_utils.search_builder('swift-proxy', 'object',
+                                            self.TEST_REMOVE_TARGET)
+        expected = 'No matching devices found'
+        self.assertEqual(result.strip('\n'), expected)
+        self.assertTrue(swift_utils.is_proxy_ring_up_to_date('swift-proxy',
+                                                             'object'))
+        self.assertTrue(
+            swift_utils.is_ring_synced('swift-proxy', 'object',
+                                       self.TEST_EXPECTED_RING_HOSTS))
 
 
 class SwiftStorageTests(test_utils.OpenStackBaseTest):

zaza/openstack/utilities/swift.py

@@ -298,3 +298,69 @@ def create_object(swift_client, proxy_app, storage_topology, resource_prefix,
         storage_topology,
         model_name=model_name)
     return container_name, object_name, obj_replicas
+
+
+def search_builder(proxy_app, ring, search_target, model_name=None):
+    """Run a swift-ring-builder search.
+
+    :param proxy_app: Name of proxy application
+    :type proxy_app: str
+    :param ring: Name of ring (one of: object, account, container)
+    :type ring: str
+    :param search_target: device search string (see: man swift-ring-builder)
+    :type search_target: str
+    :param model_name: Model to point environment at
+    :type model_name: str
+    :returns: stdout - full stdout output from swift-ring-builder cmd
+    :rtype: str
+    """
+    cmd = ('swift-ring-builder /etc/swift/{}.builder search {}'
+           ''.format(ring, search_target))
+    result = zaza.model.run_on_leader(proxy_app, cmd,
+                                      model_name=model_name)
+    return result['Stdout']
+
+
+def is_proxy_ring_up_to_date(proxy_app, ring, model_name=None):
+    """Check if the ring file is up-to-date with changes of the builder.
+
+    :param proxy_app: Name of proxy application
+    :type proxy_app: str
+    :param ring: Name of ring (one of: object, account, container)
+    :type ring: str
+    :param model_name: Model to point environment at
+    :type model_name: str
+    :returns: True if swift-ring-builder denotes ring.gz file is up-to-date
+    :rtype: str
+    """
+    logging.info('Checking ring file matches builder file')
+    cmd = ('swift-ring-builder /etc/swift/{}.builder | '
+           'grep "Ring file .* is"'.format(ring))
+    result = zaza.model.run_on_leader(proxy_app, cmd, model_name=model_name)
+    expected = ('Ring file /etc/swift/{}.ring.gz is up-to-date'
+                ''.format(ring))
+    return bool(result['Stdout'].strip('\n') == expected)
+
+
+def is_ring_synced(proxy_app, ring, expected_hosts, model_name=None):
+    """Check if md5sums of rings on swift-storage are synced to this proxy.
+
+    :param proxy_app: Name of proxy application
+    :type proxy_app: str
+    :param ring: Name of ring (one of: object, account, container)
+    :type ring: str
+    :param expoected_hosts: Number of swift-storage hosts in test environment
+    :type search_target: int
+    :param model_name: Model to point environment at
+    :type model_name: str
+    :returns: True if all expected_hosts matched md5sum of proxy ring file
+    :rtype: bool
+    """
+    logging.info('Checking ring md5sums on storage unit(s) against proxy')
+    zaza.model.block_until_all_units_idle()
+    cmd = ('swift-recon {} --md5 | '
+           'grep -A1 "ring md5" | tail -1'.format(ring))
+    result = zaza.model.run_on_leader(proxy_app, cmd, model_name=model_name)
+    expected = ('{num}/{num} hosts matched, 0 error[s] while checking hosts.'
+                ''.format(num=expected_hosts))
+    return bool(result['Stdout'].strip('\n') == expected)