# Copyright 2019 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Swift utilities.""" import logging import uuid import zaza.model import zaza.openstack.utilities.juju as juju_utils class ObjectReplica: """A replica of an object. The replica attributes show the location of an object replica. server: IP address or hostname of machine hosting replica port: Port of swift object server running on machine hosting replica device: Path to device hosting replica handoff_device: Whether this is a handoff devices. Handoff devices pass the replica on to a remote storage node. """ def __init__(self, raw_line): """Extract storage info from text.""" rl = raw_line.split() self.server, self.port = rl[2].split(':') self.device = rl[3] self.handoff_device = rl[-1] == '[Handoff]' class ObjectReplicas: """Replicas of an object.""" def __init__(self, proxy_app, account, container_name, object_name, storage_topology, model_name=None): """Find all replicas of given object. :param proxy_app: Name of proxy application :type proxy_app: str :param account: Account that owns the container. :type account: str :param container_name: Name of container that contains the object. :type container_name: str :param object_name: Name of object. :type object_name: str :param storage_topology: Dictionary keyed on IP of storage node info. :type storage_topology: {} :param model_name: Model to point environment at :type model_name: str """ self.replicas = [] self.replica_placements = {} self.storage_topology = storage_topology raw_output = self.run_get_nodes( proxy_app, account, container_name, object_name, model_name=model_name) for line in self.extract_storage_lines(raw_output): self.add_replica(line) def add_replica(self, storage_line): """Add a replica to the replica set.""" self.replicas.append(ObjectReplica(storage_line)) def extract_storage_lines(self, raw_output): """Extract replica list from output of swift-get-nodes. :param storage_line: Output of swift-get-nodes :type storage_line: str :returns: List of lines relating to replicas. :rtype: [str, ...] """ storage_lines = [] for line in raw_output.split('\n'): if line.startswith('Server:Port '): storage_lines.append(line) return storage_lines def run_get_nodes(self, proxy_app, account, container_name, object_name, model_name=None): """Run swift-get-nodes for an object on a proxy unit. :param proxy_app: Name of proxy application :type proxy_app: str :param account: Account that owns the container. :type account: str :param container_name: Name of container that contains the object. :type container_name: str :param object_name: Name of object. :type object_name: str :param model_name: Model to point environment at :type model_name: str :returns: Stdout of command :rtype: str """ ring_file = '/etc/swift/object.ring.gz' obj_cmd = "swift-get-nodes -a {} {} {} {}".format( ring_file, account, container_name, object_name) cmd_result = zaza.model.run_on_leader( proxy_app, obj_cmd, model_name=model_name) return cmd_result['Stdout'] @property def hand_off_ips(self): """Replicas which are marked as handoff devices. These are not real replicas. They hand off the replica to other node. :returns: List of IPS of handoff nodes for object. :rtype: List[str] """ return [r.server for r in self.replicas if r.handoff_device] @property def storage_ips(self): """Ip addresses of nodes that are housing a replica. :returns: List of IPS of storage nodes holding a replica of the object. :rtype: [str, ...] """ return [r.server for r in self.replicas if not r.handoff_device] @property def placements(self): """Region an zone information for each replica. Zone info is in the form: [{ 'app_name': str, 'unit': juju.Unit, 'region': int, 'zone': int}, ...] :returns: List of dicts with region and zone information. :rtype: List[Dict[str, Union[str,int]]] """ return [self.storage_topology[ip] for ip in self.storage_ips] @property def distinct_regions(self): """List of distinct regions that have a replica. :returns: List of regions that have a replica :rtype: [int, ...] """ return list(set([p['region'] for p in self.placements])) @property def all_zones(self): """List of all zones that have a replica. :returns: List of tuples (region, zone) that have a replica. :rtype: List[Tuple[str, str]] """ return [(p['region'], p['zone']) for p in self.placements] @property def distinct_zones(self): """List of distinct region + zones that have a replica. :returns: List of tuples (region, zone) that have a replica. :rtype: [(r1, z1), ...] """ return list(set(self.all_zones)) def get_swift_storage_topology(model_name=None): """Get details of storage nodes and which region and zones they belong in. :param model_name: Model to point environment at :type model_name: str :returns: Dictionary of storage nodes and their region/zone information. :rtype: { 'ip (str)': { 'app_name': str, 'unit': juju.Unit 'region': int, 'zone': int}, ...} """ topology = {} status = juju_utils.get_full_juju_status(model_name=model_name) for app_name, app_dep_config in status.applications.items(): if 'swift-storage' in app_dep_config['charm']: app_config = zaza.model.get_application_config( app_name, model_name=model_name) region = app_config['storage-region']['value'] zone = app_config['zone']['value'] for unit in zaza.model.get_units(app_name, model_name=model_name): topology[unit.public_address] = { 'app_name': app_name, 'unit': unit, 'region': region, 'zone': zone} return topology def setup_test_container(swift_client, resource_prefix): """Create a swift container for use be tests. :param swift_client: Swift client to use for object creation :type swift_client: swiftclient.Client :returns: (container_name, account_name) Container name and account name for new container :rtype: Tuple[str, str] """ run_id = str(uuid.uuid1()).split('-')[0] container_name = '{}-{}-container'.format(resource_prefix, run_id) swift_client.put_container(container_name) resp_headers, containers = swift_client.get_account() account = resp_headers['x-account-project-domain-id'] return container_name, account def apply_proxy_config(proxy_app, config, model_name=None): """Update the give proxy_app with new charm config. :param proxy_app: Name of proxy application :type proxy_app: str :param config: Dictionary of configuration setting(s) to apply :type config: dict :param model_name: Name of model to query. :type model_name: str """ current_config = zaza.model.get_application_config( proxy_app, model_name=model_name) # Although there is no harm in applying config that is a noop it # does affect the expected behaviour afterwards. So, only apply # genuine changes so we can safely expect the charm to fire a hook. for key, value in config.items(): if str(config[key]) != str(current_config[key]['value']): break else: logging.info( 'Config update for {} not required.'.format(proxy_app)) return logging.info('Updating {} charm settings'.format(proxy_app)) zaza.model.set_application_config( proxy_app, config, model_name=model_name) zaza.model.block_until_all_units_idle() def create_object(swift_client, proxy_app, storage_topology, resource_prefix, model_name=None): """Create a test object in a new container. :param swift_client: Swift client to use for object creation :type swift_client: swiftclient.Client :param proxy_app: Name of proxy application :type proxy_app: str :param storage_topology: Dictionary keyed on IP of storage node info. :type storage_topology: {} :param resource_prefix: Prefix to use when naming new resources :type resource_prefix: str :param model_name: Model to point environment at :type model_name: str :returns: (container_name, object_name, object replicas) :rtype: (str, str, ObjectReplicas) """ container_name, account = setup_test_container( swift_client, resource_prefix) object_name = 'zaza_test_object.txt' swift_client.put_object( container_name, object_name, contents='File contents', content_type='text/plain' ) obj_replicas = ObjectReplicas( proxy_app, account, container_name, object_name, storage_topology, model_name=model_name) return container_name, object_name, obj_replicas