diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index 5036b4b..2a46cf7 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -73,6 +73,8 @@ class SwiftProxyTests(test_utils.OpenStackBaseTest): """Tests specific to swift proxy.""" TEST_SEARCH_TARGET = 'd0' + TEST_REMOVE_TARGET = 'd1' + TEST_EXPECTED_RING_HOSTS = 1 TEST_WEIGHT_TARGET = 999 TEST_WEIGHT_INITIAL = 100 @@ -96,8 +98,7 @@ class SwiftProxyTests(test_utils.OpenStackBaseTest): self.assertEqual(action.status, "completed") def test_904_set_weight_action_and_validate_rebalance(self): - """Set weight of device in object ring, rebalance, and - validate rings have rebalance and replicated""" + """Set weight of device in object ring.""" logging.info('Running set-weight action on leader') action = zaza.model.run_action_on_leader( 'swift-proxy', @@ -108,24 +109,13 @@ class SwiftProxyTests(test_utils.OpenStackBaseTest): self.assertEqual(action.status, "completed") logging.info('Validating builder updated as expected') - cmd = ('swift-ring-builder /etc/swift/object.builder search {}' - ''.format(self.TEST_SEARCH_TARGET)) - result = zaza.model.run_on_leader('swift-proxy', cmd) - self.assertTrue('Stdout' in result, - 'failed to retrieve swift-ring-builder weight') + result = swift_utils.search_builder('swift-proxy', 'object', + self.TEST_SEARCH_TARGET) # disk weight is the 9th field of the second line and is a float - disk_weight = result['Stdout'].split('\n')[1].split()[8] - disk_weight = int(disk_weight.split('.')[0]) + disk_weight = int(result.split('\n')[1].split()[8].split('.')[0]) self.assertEqual(disk_weight, self.TEST_WEIGHT_TARGET) - - logging.info('Checking ring has rebalanced') - cmd = ('swift-ring-builder /etc/swift/object.builder | ' - 'grep "Ring file .* is"') - result = zaza.model.run_on_leader('swift-proxy', cmd) - self.assertTrue('Stdout' in result, - 'failed to retrieve object ring status') - expected = 'Ring file /etc/swift/object.ring.gz is up-to-date' - self.assertEqual(result['Stdout'].strip('\n'), expected) + self.assertTrue(swift_utils.is_proxy_ring_up_to_date('swift-proxy', + 'object')) logging.info('Running set-weight on leader to reset weight back') action = zaza.model.run_action_on_leader( @@ -135,15 +125,30 @@ class SwiftProxyTests(test_utils.OpenStackBaseTest): 'search-value': self.TEST_SEARCH_TARGET, 'weight': self.TEST_WEIGHT_INITIAL}) self.assertEqual(action.status, "completed") + self.assertTrue( + swift_utils.is_ring_synced('swift-proxy', 'object', + self.TEST_EXPECTED_RING_HOSTS)) - logging.info('Checking ring md5sums on storage units against proxy') - zaza.model.block_until_all_units_idle() - cmd = 'swift-recon --md5 | grep -A1 "ring md5" | tail -1' - result = zaza.model.run_on_leader('swift-proxy', cmd) - self.assertTrue('Stdout' in result, - 'failed to retrieve swift-recon results') - expected = '1/1 hosts matched, 0 error[s] while checking hosts.' - self.assertEqual(result['Stdout'].strip('\n'), expected) + def test_905_remove_device_action_and_validate_rebalance(self): + """Remove device from object ring.""" + logging.info('Running remove-devices action on leader') + action = zaza.model.run_action_on_leader( + 'swift-proxy', + 'remove-devices', + action_params={'ring': 'object', + 'search-value': self.TEST_REMOVE_TARGET}) + self.assertEqual(action.status, "completed") + + logging.info('Validating builder updated as expected') + result = swift_utils.search_builder('swift-proxy', 'object', + self.TEST_REMOVE_TARGET) + expected = 'No matching devices found' + self.assertEqual(result.strip('\n'), expected) + self.assertTrue(swift_utils.is_proxy_ring_up_to_date('swift-proxy', + 'object')) + self.assertTrue( + swift_utils.is_ring_synced('swift-proxy', 'object', + self.TEST_EXPECTED_RING_HOSTS)) class SwiftStorageTests(test_utils.OpenStackBaseTest): diff --git a/zaza/openstack/utilities/swift.py b/zaza/openstack/utilities/swift.py index 2b1f46a..b44e25d 100644 --- a/zaza/openstack/utilities/swift.py +++ b/zaza/openstack/utilities/swift.py @@ -298,3 +298,69 @@ def create_object(swift_client, proxy_app, storage_topology, resource_prefix, storage_topology, model_name=model_name) return container_name, object_name, obj_replicas + + +def search_builder(proxy_app, ring, search_target, model_name=None): + """Run a swift-ring-builder search. + + :param proxy_app: Name of proxy application + :type proxy_app: str + :param ring: Name of ring (one of: object, account, container) + :type ring: str + :param search_target: device search string (see: man swift-ring-builder) + :type search_target: str + :param model_name: Model to point environment at + :type model_name: str + :returns: stdout - full stdout output from swift-ring-builder cmd + :rtype: str + """ + cmd = ('swift-ring-builder /etc/swift/{}.builder search {}' + ''.format(ring, search_target)) + result = zaza.model.run_on_leader(proxy_app, cmd, + model_name=model_name) + return result['Stdout'] + + +def is_proxy_ring_up_to_date(proxy_app, ring, model_name=None): + """Check if the ring file is up-to-date with changes of the builder. + + :param proxy_app: Name of proxy application + :type proxy_app: str + :param ring: Name of ring (one of: object, account, container) + :type ring: str + :param model_name: Model to point environment at + :type model_name: str + :returns: True if swift-ring-builder denotes ring.gz file is up-to-date + :rtype: str + """ + logging.info('Checking ring file matches builder file') + cmd = ('swift-ring-builder /etc/swift/{}.builder | ' + 'grep "Ring file .* is"'.format(ring)) + result = zaza.model.run_on_leader(proxy_app, cmd, model_name=model_name) + expected = ('Ring file /etc/swift/{}.ring.gz is up-to-date' + ''.format(ring)) + return bool(result['Stdout'].strip('\n') == expected) + + +def is_ring_synced(proxy_app, ring, expected_hosts, model_name=None): + """Check if md5sums of rings on swift-storage are synced to this proxy. + + :param proxy_app: Name of proxy application + :type proxy_app: str + :param ring: Name of ring (one of: object, account, container) + :type ring: str + :param expoected_hosts: Number of swift-storage hosts in test environment + :type search_target: int + :param model_name: Model to point environment at + :type model_name: str + :returns: True if all expected_hosts matched md5sum of proxy ring file + :rtype: bool + """ + logging.info('Checking ring md5sums on storage unit(s) against proxy') + zaza.model.block_until_all_units_idle() + cmd = ('swift-recon {} --md5 | ' + 'grep -A1 "ring md5" | tail -1'.format(ring)) + result = zaza.model.run_on_leader(proxy_app, cmd, model_name=model_name) + expected = ('{num}/{num} hosts matched, 0 error[s] while checking hosts.' + ''.format(num=expected_hosts)) + return bool(result['Stdout'].strip('\n') == expected)