diff --git a/unit_tests/utilities/swift_test_data.py b/unit_tests/utilities/swift_test_data.py new file mode 100644 index 0000000..ac3145a --- /dev/null +++ b/unit_tests/utilities/swift_test_data.py @@ -0,0 +1,69 @@ +# flake8: noqa + +SWIFT_GET_NODES_STDOUT = """ +Account 23934cb1850c4d28b1ca113a24c0e46b +Container zaza-swift-gr-tests-f3129278-container +Object zaza_test_object.txt + + +Partition 146 +Hash 928c2f8006efeeb4b1164f4cce035887 + +Server:Port Device 10.5.0.38:6000 loop0 +Server:Port Device 10.5.0.4:6000 loop0 +Server:Port Device 10.5.0.9:6000 loop0 [Handoff] +Server:Port Device 10.5.0.34:6000 loop0 [Handoff] +Server:Port Device 10.5.0.15:6000 loop0 [Handoff] +Server:Port Device 10.5.0.18:6000 loop0 [Handoff] + + +curl -g -I -XHEAD "http://10.5.0.38:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" +curl -g -I -XHEAD "http://10.5.0.4:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" +curl -g -I -XHEAD "http://10.5.0.9:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff] +curl -g -I -XHEAD "http://10.5.0.34:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff] +curl -g -I -XHEAD "http://10.5.0.15:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff] +curl -g -I -XHEAD "http://10.5.0.18:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff] + + +Use your own device location of servers: +such as "export DEVICE=/srv/node" +ssh 10.5.0.38 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" +ssh 10.5.0.4 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" +ssh 10.5.0.9 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff] +ssh 10.5.0.34 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff] +ssh 10.5.0.15 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff] +ssh 10.5.0.18 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff] + +note: `/srv/node*` is used as default value of `devices`, the real value is set in the config file on each storage node. +""" + +STORAGE_TOPOLOGY = { + '10.5.0.18': { + 'app_name': 'swift-storage-region1-zone1', + 'unit': "swift-storage-region1-zone1/0", + 'region': 1, + 'zone': 1}, + '10.5.0.34': { + 'app_name': 'swift-storage-region1-zone2', + 'unit': "swift-storage-region1-zone2/0", + 'region': 1, + 'zone': 2}, + '10.5.0.4': { + 'app_name': 'swift-storage-region1-zone3', + 'unit': "swift-storage-region1-zone3/0", + 'region': 1, + 'zone': 3}, + '10.5.0.9': { + 'app_name': 'swift-storage-region2-zone1', + 'unit': "swift-storage-region2-zone1/0", + 'region': 2, + 'zone': 1}, + '10.5.0.15': { + 'app_name': 'swift-storage-region2-zone2', + 'unit': "swift-storage-region2-zone2/0", + 'region': 2, 'zone': 2}, + '10.5.0.38': { + 'app_name': 'swift-storage-region2-zone3', + 'unit': "swift-storage-region2-zone3/0", + 'region': 2, + 'zone': 3}} diff --git a/unit_tests/utilities/test_zaza_utilities_swift.py b/unit_tests/utilities/test_zaza_utilities_swift.py new file mode 100644 index 0000000..8c87556 --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_swift.py @@ -0,0 +1,187 @@ +import copy +import mock +import unit_tests.utils as ut_utils +import uuid + +import zaza.model +import zaza.openstack.utilities.swift as swift_utils +import zaza.openstack.utilities.juju as juju_utils + +import unit_tests.utilities.swift_test_data as swift_test_data + + +class TestSwiftUtils(ut_utils.BaseTestCase): + + def setUp(self): + super(TestSwiftUtils, self).setUp() + + def test_ObjectReplica_init(self): + obj_rep = swift_utils.ObjectReplica( + "Server:Port Device 10.5.0.38:6000 loop0") + self.assertEqual( + obj_rep.server, + "10.5.0.38") + self.assertEqual( + obj_rep.port, + "6000") + self.assertEqual( + obj_rep.device, + "loop0") + self.assertFalse(obj_rep.handoff_device) + obj_rep = swift_utils.ObjectReplica( + "Server:Port Device 10.5.0.9:6000 loop0 [Handoff]") + self.assertTrue(obj_rep.handoff_device) + + def test_ObjectReplicas(self): + self.patch_object(zaza.model, 'run_on_leader') + self.run_on_leader.return_value = { + 'Stdout': swift_test_data.SWIFT_GET_NODES_STDOUT} + obj_replicas = swift_utils.ObjectReplicas( + 'swift-proxy-region1', + 'account123', + 'my-container', + 'my-object', + swift_test_data.STORAGE_TOPOLOGY, + 'my-model') + self.assertEqual( + sorted(obj_replicas.hand_off_ips), + ['10.5.0.15', '10.5.0.18', '10.5.0.34', '10.5.0.9']) + self.assertEqual( + sorted(obj_replicas.storage_ips), + ['10.5.0.38', '10.5.0.4']) + self.assertEqual( + obj_replicas.placements, + [ + { + 'app_name': 'swift-storage-region2-zone3', + 'region': 2, + 'unit': 'swift-storage-region2-zone3/0', + 'zone': 3}, + { + 'app_name': 'swift-storage-region1-zone3', + 'region': 1, + 'unit': 'swift-storage-region1-zone3/0', + 'zone': 3}]) + self.assertEqual( + obj_replicas.distinct_regions, + [1, 2]) + self.assertEqual( + sorted(obj_replicas.all_zones), + [(1, 3), (2, 3)]) + self.assertEqual( + sorted(obj_replicas.distinct_zones), + [(1, 3), (2, 3)]) + + def test_get_swift_storage_topology(self): + unit_r1z1_mock = mock.MagicMock(public_address='10.5.0.18') + unit_r1z2_mock = mock.MagicMock(public_address='10.5.0.34') + unit_r1z3_mock = mock.MagicMock(public_address='10.5.0.4') + unit_r2z1_mock = mock.MagicMock(public_address='10.5.0.9') + unit_r2z2_mock = mock.MagicMock(public_address='10.5.0.15') + unit_r2z3_mock = mock.MagicMock(public_address='10.5.0.38') + app_units = { + 'swift-storage-region1-zone1': [unit_r1z1_mock], + 'swift-storage-region1-zone2': [unit_r1z2_mock], + 'swift-storage-region1-zone3': [unit_r1z3_mock], + 'swift-storage-region2-zone1': [unit_r2z1_mock], + 'swift-storage-region2-zone2': [unit_r2z2_mock], + 'swift-storage-region2-zone3': [unit_r2z3_mock]} + + expected_topology = copy.deepcopy(swift_test_data.STORAGE_TOPOLOGY) + self.patch_object(juju_utils, 'get_full_juju_status') + self.patch_object(zaza.model, 'get_application_config') + self.patch_object(zaza.model, 'get_units') + juju_status = mock.MagicMock() + juju_status.applications = {} + self.get_full_juju_status.return_value = juju_status + + for app_name, units in app_units.items(): + expected_topology[units[0].public_address]['unit'] = units[0] + + app_config = {} + for app_name in app_units.keys(): + juju_status.applications[app_name] = {'charm': 'cs:swift-storage'} + region = int(app_name.split('-')[2].replace('region', '')) + zone = int(app_name.split('-')[3].replace('zone', '')) + app_config[app_name] = { + 'region': {'value': region}, + 'zone': {'value': zone}} + + self.get_application_config.side_effect = \ + lambda x, model_name: app_config[x] + self.get_units.side_effect = lambda x, model_name: app_units[x] + self.assertEqual( + swift_utils.get_swift_storage_topology(), + expected_topology) + + def test_setup_test_container(self): + swift_client = mock.MagicMock() + self.patch_object(uuid, 'uuid1', return_value='auuid') + swift_client.get_account.return_value = ( + {'x-account-project-domain-id': 'domain-id'}, + 'bob-auuid-container') + self.assertEqual( + swift_utils.setup_test_container(swift_client, 'bob'), + ('bob-auuid-container', 'domain-id')) + swift_client.put_container.assert_called_once_with( + 'bob-auuid-container') + + def test_apply_proxy_config(self): + self.patch_object(zaza.model, 'block_until_all_units_idle') + self.patch_object( + zaza.model, + 'get_application_config', + return_value={ + 'go-faster': { + 'value': False}}) + self.patch_object(zaza.model, 'set_application_config') + swift_utils.apply_proxy_config( + 'proxy-app', + {'go-faster': True}) + self.set_application_config.assert_called_once_with( + 'proxy-app', {'go-faster': True}, model_name=None) + + def test_apply_proxy_config_noop(self): + self.patch_object(zaza.model, 'block_until_all_units_idle') + self.patch_object( + zaza.model, + 'get_application_config', + return_value={ + 'go-faster': { + 'value': True}}) + self.patch_object(zaza.model, 'set_application_config') + swift_utils.apply_proxy_config( + 'proxy-app', + {'go-faster': True}) + self.assertFalse(self.set_application_config.called) + + def test_create_object(self): + self.patch_object(swift_utils, 'setup_test_container') + self.setup_test_container.return_value = ('new-container', 'domain-id') + self.patch_object( + swift_utils, + 'ObjectReplicas', + return_value='obj_replicas') + swift_client = mock.MagicMock() + self.assertEqual( + swift_utils.create_object( + swift_client, + 'proxy-app', + swift_test_data.STORAGE_TOPOLOGY, + 'my-prefix'), + ('new-container', 'zaza_test_object.txt', 'obj_replicas')) + self.setup_test_container.assert_called_once_with( + swift_client, + 'my-prefix') + swift_client.put_object.assert_called_once_with( + 'new-container', + 'zaza_test_object.txt', + content_type='text/plain', + contents='File contents') + self.ObjectReplicas.assert_called_once_with( + 'proxy-app', + 'domain-id', + 'new-container', + 'zaza_test_object.txt', + swift_test_data.STORAGE_TOPOLOGY, + model_name=None) diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index ad2c0e5..a62d56f 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -17,11 +17,14 @@ """Encapsulate swift testing.""" import logging +import tenacity +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.configure.guest import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.utilities.swift as swift_utils class SwiftImageCreateTest(test_utils.OpenStackBaseTest): @@ -110,3 +113,115 @@ class SwiftStorageTests(test_utils.OpenStackBaseTest): 'swift-container-sync'] with self.pause_resume(services): logging.info("Testing pause resume") + + +class SwiftGlobalReplicationTests(test_utils.OpenStackBaseTest): + """Test swift global replication.""" + + RESOURCE_PREFIX = 'zaza-swift-gr-tests' + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + cls.region1_model_alias = 'swift_gr_region1' + cls.region1_proxy_app = 'swift-proxy-region1' + cls.region2_model_alias = 'swift_gr_region2' + cls.region2_proxy_app = 'swift-proxy-region2' + super(SwiftGlobalReplicationTests, cls).setUpClass( + application_name=cls.region1_proxy_app, + model_alias=cls.region1_model_alias) + cls.region1_model_name = cls.model_aliases[cls.region1_model_alias] + cls.region2_model_name = cls.model_aliases[cls.region2_model_alias] + cls.storage_topology = swift_utils.get_swift_storage_topology( + model_name=cls.region1_model_name) + cls.storage_topology.update( + swift_utils.get_swift_storage_topology( + model_name=cls.region2_model_name)) + cls.swift_session = openstack_utils.get_keystone_session_from_relation( + cls.region1_proxy_app, + model_name=cls.region1_model_name) + cls.swift_region1 = openstack_utils.get_swift_session_client( + cls.swift_session, + region_name='RegionOne') + cls.swift_region2 = openstack_utils.get_swift_session_client( + cls.swift_session, + region_name='RegionTwo') + + @classmethod + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=16, max=600), + reraise=True, + stop=tenacity.stop_after_attempt(10)) + def tearDown(cls): + """Remove test resources. + + The retry decotator is needed as the deletes are async so objects + are sometime not fully deleted before their container. + """ + logging.info('Running teardown') + resp_headers, containers = cls.swift_region1.get_account() + logging.info('Found containers {}'.format(containers)) + for container in containers: + if not container['name'].startswith(cls.RESOURCE_PREFIX): + continue + for obj in cls.swift_region1.get_container(container['name'])[1]: + logging.info('Deleting object {} from {}'.format( + obj['name'], + container['name'])) + cls.swift_region1.delete_object( + container['name'], + obj['name']) + logging.info('Deleting container {}'.format(container['name'])) + cls.swift_region1.delete_container(container['name']) + + def test_two_regions_any_zones_two_replicas(self): + """Create an object with two replicas across two regions.""" + swift_utils.apply_proxy_config( + self.region1_proxy_app, + { + 'write-affinity': 'r1, r2', + 'write-affinity-node-count': '1', + 'replicas': '2'}, + self.region1_model_name) + container_name, obj_name, obj_replicas = swift_utils.create_object( + self.swift_region1, + self.region1_proxy_app, + self.storage_topology, + self.RESOURCE_PREFIX, + model_name=self.region1_model_name) + # Check object is accessible from other regions proxy. + self.swift_region2.head_object(container_name, obj_name) + # Check there is at least one replica in each region. + self.assertEqual( + sorted(obj_replicas.distinct_regions), + [1, 2]) + # Check there are two relicas + self.assertEqual( + len(obj_replicas.all_zones), + 2) + + def test_two_regions_any_zones_three_replicas(self): + """Create an object with three replicas across two regions.""" + swift_utils.apply_proxy_config( + self.region1_proxy_app, + { + 'write-affinity': 'r1, r2', + 'write-affinity-node-count': '1', + 'replicas': '3'}, + self.region1_model_name) + container_name, obj_name, obj_replicas = swift_utils.create_object( + self.swift_region1, + self.region1_proxy_app, + self.storage_topology, + self.RESOURCE_PREFIX, + model_name=self.region1_model_name) + # Check object is accessible from other regions proxy. + self.swift_region2.head_object(container_name, obj_name) + # Check there is at least one replica in each region. + self.assertEqual( + sorted(obj_replicas.distinct_regions), + [1, 2]) + # Check there are three relicas + self.assertEqual( + len(obj_replicas.all_zones), + 3) diff --git a/zaza/openstack/utilities/swift.py b/zaza/openstack/utilities/swift.py new file mode 100644 index 0000000..43a977d --- /dev/null +++ b/zaza/openstack/utilities/swift.py @@ -0,0 +1,295 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Swift utilities.""" + +import logging +import uuid +import zaza.model +import zaza.openstack.utilities.juju as juju_utils + + +class ObjectReplica: + """A replica of an object. + + The replica attributes show the location of an object replica. + + server: IP address or hostname of machine hosting replica + port: Port of swift object server running on machine hosting replica + device: Path to device hosting replica + handoff_device: Whether this is a handoff devices. Handoff devices pass + the replica on to a remote storage node. + """ + + def __init__(self, raw_line): + """Extract storage info from text.""" + rl = raw_line.split() + self.server, self.port = rl[2].split(':') + self.device = rl[3] + self.handoff_device = rl[-1] == '[Handoff]' + + +class ObjectReplicas: + """Replicas of an object.""" + + def __init__(self, proxy_app, account, container_name, object_name, + storage_topology, model_name=None): + """Find all replicas of given object. + + :param proxy_app: Name of proxy application + :type proxy_app: str + :param account: Account that owns the container. + :type account: str + :param container_name: Name of container that contains the object. + :type container_name: str + :param object_name: Name of object. + :type object_name: str + :param storage_topology: Dictionary keyed on IP of storage node info. + :type storage_topology: {} + :param model_name: Model to point environment at + :type model_name: str + """ + self.replicas = [] + self.replica_placements = {} + self.storage_topology = storage_topology + raw_output = self.run_get_nodes( + proxy_app, + account, + container_name, + object_name, + model_name=model_name) + for line in self.extract_storage_lines(raw_output): + self.add_replica(line) + + def add_replica(self, storage_line): + """Add a replica to the replica set.""" + self.replicas.append(ObjectReplica(storage_line)) + + def extract_storage_lines(self, raw_output): + """Extract replica list from output of swift-get-nodes. + + :param storage_line: Output of swift-get-nodes + :type storage_line: str + :returns: List of lines relating to replicas. + :rtype: [str, ...] + """ + storage_lines = [] + for line in raw_output.split('\n'): + if line.startswith('Server:Port '): + storage_lines.append(line) + return storage_lines + + def run_get_nodes(self, proxy_app, account, container_name, object_name, + model_name=None): + """Run swift-get-nodes for an object on a proxy unit. + + :param proxy_app: Name of proxy application + :type proxy_app: str + :param account: Account that owns the container. + :type account: str + :param container_name: Name of container that contains the object. + :type container_name: str + :param object_name: Name of object. + :type object_name: str + :param model_name: Model to point environment at + :type model_name: str + """ + ring_file = '/etc/swift/object.ring.gz' + obj_cmd = "swift-get-nodes -a {} {} {} {}".format( + ring_file, + account, + container_name, + object_name) + cmd_result = zaza.model.run_on_leader( + proxy_app, + obj_cmd, + model_name=model_name) + return cmd_result['Stdout'] + + @property + def hand_off_ips(self): + """Replicas which are marked as handoff devices. + + These are not real replicas. They hand off the replica to other node. + + :returns: List of IPS of handoff nodes for object. + :rtype: [str, ...] + """ + return [r.server for r in self.replicas if r.handoff_device] + + @property + def storage_ips(self): + """Ip addresses of nodes that are housing a replica. + + :returns: List of IPS of storage nodes holding a replica of the object. + :rtype: [str, ...] + """ + return [r.server for r in self.replicas if not r.handoff_device] + + @property + def placements(self): + """Region an zone information for each replica. + + :returns: List of dicts with region and zone information. + :rtype: [{ + 'app_name': str, + 'unit': juju.Unit, + 'region': int, + 'zone': int}, ...] + """ + return [self.storage_topology[ip] for ip in self.storage_ips] + + @property + def distinct_regions(self): + """List of distinct regions that have a replica. + + :returns: List of regions that have a replica + :rtype: [int, ...] + """ + return list(set([p['region'] for p in self.placements])) + + @property + def all_zones(self): + """List of all zones that have a replica. + + :returns: List of tuples (region, zone) that have a replica. + :rtype: [(r1, z1), ...] + """ + return [(p['region'], p['zone']) for p in self.placements] + + @property + def distinct_zones(self): + """List of distinct region + zones that have a replica. + + :returns: List of tuples (region, zone) that have a replica. + :rtype: [(r1, z1), ...] + """ + return list(set(self.all_zones)) + + +def get_swift_storage_topology(model_name=None): + """Get details of storage nodes and which region and zones they belong in. + + :param model_name: Model to point environment at + :type model_name: str + :returns: Dictionary of storage nodes and their region/zone information. + :rtype: { + 'ip (str)': { + 'app_name': str, + 'unit': juju.Unit + 'region': int, + 'zone': int}, + ...} + """ + topology = {} + status = juju_utils.get_full_juju_status(model_name=model_name) + for app_name, app_dep_config in status.applications.items(): + if 'swift-storage' in app_dep_config['charm']: + app_config = zaza.model.get_application_config( + app_name, + model_name=model_name) + region = app_config['region']['value'] + zone = app_config['zone']['value'] + for unit in zaza.model.get_units(app_name, model_name=model_name): + topology[unit.public_address] = { + 'app_name': app_name, + 'unit': unit, + 'region': region, + 'zone': zone} + return topology + + +def setup_test_container(swift_client, resource_prefix): + """Create a swift container for use be tests. + + :param swift_client: Swift client to use for object creation + :type swift_client: swiftclient.Client + :returns: (container_name, account_name) Container name and account + name for new container + :rtype: (str, str) + """ + run_id = str(uuid.uuid1()).split('-')[0] + container_name = '{}-{}-container'.format(resource_prefix, run_id) + swift_client.put_container(container_name) + resp_headers, containers = swift_client.get_account() + account = resp_headers['x-account-project-domain-id'] + return container_name, account + + +def apply_proxy_config(proxy_app, config, model_name=None): + """Update the give proxy_app with new charm config. + + :param proxy_app: Name of proxy application + :type proxy_app: str + :param config: Dictionary of configuration setting(s) to apply + :type config: dict + :param model_name: Name of model to query. + :type model_name: str + """ + current_config = zaza.model.get_application_config( + proxy_app, + model_name=model_name) + # Although there is no harm in applying config that is a noop it + # does affect the expected behaviour afterwards. So, only apply + # genuine changes so we can safely expect the charm to fire a hook. + for key, value in config.items(): + if str(config[key]) != str(current_config[key]['value']): + break + else: + logging.info( + 'Config update for {} not required.'.format(proxy_app)) + return + logging.info('Updating {} charm settings'.format(proxy_app)) + zaza.model.set_application_config( + proxy_app, + config, + model_name=model_name) + zaza.model.block_until_all_units_idle() + + +def create_object(swift_client, proxy_app, storage_topology, resource_prefix, + model_name=None): + """Create a test object in a new container. + + :param swift_client: Swift client to use for object creation + :type swift_client: swiftclient.Client + :param proxy_app: Name of proxy application + :type proxy_app: str + :param storage_topology: Dictionary keyed on IP of storage node info. + :type storage_topology: {} + :param resource_prefix: Prefix to use when naming new resources + :type resource_prefix: str + :param model_name: Model to point environment at + :type model_name: str + :returns: (container_name, object_name, object replicas) + :rtype: (str, str, ObjectReplicas) + """ + container_name, account = setup_test_container( + swift_client, + resource_prefix) + object_name = 'zaza_test_object.txt' + swift_client.put_object( + container_name, + object_name, + contents='File contents', + content_type='text/plain' + ) + obj_replicas = ObjectReplicas( + proxy_app, + account, + container_name, + object_name, + storage_topology, + model_name=model_name) + return container_name, object_name, obj_replicas