arif/zaza-openstack-tests
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Swift Global Replication Tests

Browse Source
This commit is contained in:
Liam Young
2019-11-29 09:31:05 +00:00
parent 5a65e05c37
commit d2904e228d
4 changed files with 666 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
unit_tests/utilities/swift_test_data.py Normal file
View File

@@ -0,0 +1,69 @@
# flake8: noqa
SWIFT_GET_NODES_STDOUT = """
Account 23934cb1850c4d28b1ca113a24c0e46b
Container zaza-swift-gr-tests-f3129278-container
Object zaza_test_object.txt
Partition 146
Hash 928c2f8006efeeb4b1164f4cce035887
Server:Port Device 10.5.0.38:6000 loop0
Server:Port Device 10.5.0.4:6000 loop0
Server:Port Device 10.5.0.9:6000 loop0 [Handoff]
Server:Port Device 10.5.0.34:6000 loop0 [Handoff]
Server:Port Device 10.5.0.15:6000 loop0 [Handoff]
Server:Port Device 10.5.0.18:6000 loop0 [Handoff]
curl -g -I -XHEAD "http://10.5.0.38:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt"
curl -g -I -XHEAD "http://10.5.0.4:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt"
curl -g -I -XHEAD "http://10.5.0.9:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff]
curl -g -I -XHEAD "http://10.5.0.34:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff]
curl -g -I -XHEAD "http://10.5.0.15:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff]
curl -g -I -XHEAD "http://10.5.0.18:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff]
Use your own device location of servers:
such as "export DEVICE=/srv/node"
ssh 10.5.0.38 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887"
ssh 10.5.0.4 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887"
ssh 10.5.0.9 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff]
ssh 10.5.0.34 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff]
ssh 10.5.0.15 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff]
ssh 10.5.0.18 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff]
note: `/srv/node*` is used as default value of `devices`, the real value is set in the config file on each storage node.
"""
STORAGE_TOPOLOGY = {
'10.5.0.18': {
'app_name': 'swift-storage-region1-zone1',
'unit': "swift-storage-region1-zone1/0",
'region': 1,
'zone': 1},
'10.5.0.34': {
'app_name': 'swift-storage-region1-zone2',
'unit': "swift-storage-region1-zone2/0",
'region': 1,
'zone': 2},
'10.5.0.4': {
'app_name': 'swift-storage-region1-zone3',
'unit': "swift-storage-region1-zone3/0",
'region': 1,
'zone': 3},
'10.5.0.9': {
'app_name': 'swift-storage-region2-zone1',
'unit': "swift-storage-region2-zone1/0",
'region': 2,
'zone': 1},
'10.5.0.15': {
'app_name': 'swift-storage-region2-zone2',
'unit': "swift-storage-region2-zone2/0",
'region': 2, 'zone': 2},
'10.5.0.38': {
'app_name': 'swift-storage-region2-zone3',
'unit': "swift-storage-region2-zone3/0",
'region': 2,
'zone': 3}}

187
unit_tests/utilities/test_zaza_utilities_swift.py Normal file
View File

@@ -0,0 +1,187 @@
import copy
import mock
import unit_tests.utils as ut_utils
import uuid
import zaza.model
import zaza.openstack.utilities.swift as swift_utils
import zaza.openstack.utilities.juju as juju_utils
import unit_tests.utilities.swift_test_data as swift_test_data
class TestSwiftUtils(ut_utils.BaseTestCase):
def setUp(self):
super(TestSwiftUtils, self).setUp()
def test_ObjectReplica_init(self):
obj_rep = swift_utils.ObjectReplica(
"Server:Port Device 10.5.0.38:6000 loop0")
self.assertEqual(
obj_rep.server,
"10.5.0.38")
self.assertEqual(
obj_rep.port,
"6000")
self.assertEqual(
obj_rep.device,
"loop0")
self.assertFalse(obj_rep.handoff_device)
obj_rep = swift_utils.ObjectReplica(
"Server:Port Device 10.5.0.9:6000 loop0 [Handoff]")
self.assertTrue(obj_rep.handoff_device)
def test_ObjectReplicas(self):
self.patch_object(zaza.model, 'run_on_leader')
self.run_on_leader.return_value = {
'Stdout': swift_test_data.SWIFT_GET_NODES_STDOUT}
obj_replicas = swift_utils.ObjectReplicas(
'swift-proxy-region1',
'account123',
'my-container',
'my-object',
swift_test_data.STORAGE_TOPOLOGY,
'my-model')
self.assertEqual(
sorted(obj_replicas.hand_off_ips),
['10.5.0.15', '10.5.0.18', '10.5.0.34', '10.5.0.9'])
self.assertEqual(
sorted(obj_replicas.storage_ips),
['10.5.0.38', '10.5.0.4'])
self.assertEqual(
obj_replicas.placements,
[
{
'app_name': 'swift-storage-region2-zone3',
'region': 2,
'unit': 'swift-storage-region2-zone3/0',
'zone': 3},
{
'app_name': 'swift-storage-region1-zone3',
'region': 1,
'unit': 'swift-storage-region1-zone3/0',
'zone': 3}])
self.assertEqual(
obj_replicas.distinct_regions,
[1, 2])
self.assertEqual(
sorted(obj_replicas.all_zones),
[(1, 3), (2, 3)])
self.assertEqual(
sorted(obj_replicas.distinct_zones),
[(1, 3), (2, 3)])
def test_get_swift_storage_topology(self):
unit_r1z1_mock = mock.MagicMock(public_address='10.5.0.18')
unit_r1z2_mock = mock.MagicMock(public_address='10.5.0.34')
unit_r1z3_mock = mock.MagicMock(public_address='10.5.0.4')
unit_r2z1_mock = mock.MagicMock(public_address='10.5.0.9')
unit_r2z2_mock = mock.MagicMock(public_address='10.5.0.15')
unit_r2z3_mock = mock.MagicMock(public_address='10.5.0.38')
app_units = {
'swift-storage-region1-zone1': [unit_r1z1_mock],
'swift-storage-region1-zone2': [unit_r1z2_mock],
'swift-storage-region1-zone3': [unit_r1z3_mock],
'swift-storage-region2-zone1': [unit_r2z1_mock],
'swift-storage-region2-zone2': [unit_r2z2_mock],
'swift-storage-region2-zone3': [unit_r2z3_mock]}
expected_topology = copy.deepcopy(swift_test_data.STORAGE_TOPOLOGY)
self.patch_object(juju_utils, 'get_full_juju_status')
self.patch_object(zaza.model, 'get_application_config')
self.patch_object(zaza.model, 'get_units')
juju_status = mock.MagicMock()
juju_status.applications = {}
self.get_full_juju_status.return_value = juju_status
for app_name, units in app_units.items():
expected_topology[units[0].public_address]['unit'] = units[0]
app_config = {}
for app_name in app_units.keys():
juju_status.applications[app_name] = {'charm': 'cs:swift-storage'}
region = int(app_name.split('-')[2].replace('region', ''))
zone = int(app_name.split('-')[3].replace('zone', ''))
app_config[app_name] = {
'region': {'value': region},
'zone': {'value': zone}}
self.get_application_config.side_effect = \
lambda x, model_name: app_config[x]
self.get_units.side_effect = lambda x, model_name: app_units[x]
self.assertEqual(
swift_utils.get_swift_storage_topology(),
expected_topology)
def test_setup_test_container(self):
swift_client = mock.MagicMock()
self.patch_object(uuid, 'uuid1', return_value='auuid')
swift_client.get_account.return_value = (
{'x-account-project-domain-id': 'domain-id'},
'bob-auuid-container')
self.assertEqual(
swift_utils.setup_test_container(swift_client, 'bob'),
('bob-auuid-container', 'domain-id'))
swift_client.put_container.assert_called_once_with(
'bob-auuid-container')
def test_apply_proxy_config(self):
self.patch_object(zaza.model, 'block_until_all_units_idle')
self.patch_object(
zaza.model,
'get_application_config',
return_value={
'go-faster': {
'value': False}})
self.patch_object(zaza.model, 'set_application_config')
swift_utils.apply_proxy_config(
'proxy-app',
{'go-faster': True})
self.set_application_config.assert_called_once_with(
'proxy-app', {'go-faster': True}, model_name=None)
def test_apply_proxy_config_noop(self):
self.patch_object(zaza.model, 'block_until_all_units_idle')
self.patch_object(
zaza.model,
'get_application_config',
return_value={
'go-faster': {
'value': True}})
self.patch_object(zaza.model, 'set_application_config')
swift_utils.apply_proxy_config(
'proxy-app',
{'go-faster': True})
self.assertFalse(self.set_application_config.called)
def test_create_object(self):
self.patch_object(swift_utils, 'setup_test_container')
self.setup_test_container.return_value = ('new-container', 'domain-id')
self.patch_object(
swift_utils,
'ObjectReplicas',
return_value='obj_replicas')
swift_client = mock.MagicMock()
self.assertEqual(
swift_utils.create_object(
swift_client,
'proxy-app',
swift_test_data.STORAGE_TOPOLOGY,
'my-prefix'),
('new-container', 'zaza_test_object.txt', 'obj_replicas'))
self.setup_test_container.assert_called_once_with(
swift_client,
'my-prefix')
swift_client.put_object.assert_called_once_with(
'new-container',
'zaza_test_object.txt',
content_type='text/plain',
contents='File contents')
self.ObjectReplicas.assert_called_once_with(
'proxy-app',
'domain-id',
'new-container',
'zaza_test_object.txt',
swift_test_data.STORAGE_TOPOLOGY,
model_name=None)

115
zaza/openstack/charm_tests/swift/tests.py
View File

@@ -17,11 +17,14 @@
"""Encapsulate swift testing."""
import logging
import tenacity
import zaza.model
import zaza.openstack.charm_tests.test_utils as test_utils
import zaza.openstack.charm_tests.glance.setup as glance_setup
import zaza.openstack.configure.guest
import zaza.openstack.utilities.openstack as openstack_utils
import zaza.openstack.utilities.swift as swift_utils
class SwiftImageCreateTest(test_utils.OpenStackBaseTest):
@@ -110,3 +113,115 @@ class SwiftStorageTests(test_utils.OpenStackBaseTest):
'swift-container-sync']
with self.pause_resume(services):
logging.info("Testing pause resume")
class SwiftGlobalReplicationTests(test_utils.OpenStackBaseTest):
"""Test swift global replication."""
RESOURCE_PREFIX = 'zaza-swift-gr-tests'
@classmethod
def setUpClass(cls):
"""Run class setup for running tests."""
cls.region1_model_alias = 'swift_gr_region1'
cls.region1_proxy_app = 'swift-proxy-region1'
cls.region2_model_alias = 'swift_gr_region2'
cls.region2_proxy_app = 'swift-proxy-region2'
super(SwiftGlobalReplicationTests, cls).setUpClass(
application_name=cls.region1_proxy_app,
model_alias=cls.region1_model_alias)
cls.region1_model_name = cls.model_aliases[cls.region1_model_alias]
cls.region2_model_name = cls.model_aliases[cls.region2_model_alias]
cls.storage_topology = swift_utils.get_swift_storage_topology(
model_name=cls.region1_model_name)
cls.storage_topology.update(
swift_utils.get_swift_storage_topology(
model_name=cls.region2_model_name))
cls.swift_session = openstack_utils.get_keystone_session_from_relation(
cls.region1_proxy_app,
model_name=cls.region1_model_name)
cls.swift_region1 = openstack_utils.get_swift_session_client(
cls.swift_session,
region_name='RegionOne')
cls.swift_region2 = openstack_utils.get_swift_session_client(
cls.swift_session,
region_name='RegionTwo')
@classmethod
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=1, min=16, max=600),
reraise=True,
stop=tenacity.stop_after_attempt(10))
def tearDown(cls):
"""Remove test resources.
The retry decotator is needed as the deletes are async so objects
are sometime not fully deleted before their container.
"""
logging.info('Running teardown')
resp_headers, containers = cls.swift_region1.get_account()
logging.info('Found containers {}'.format(containers))
for container in containers:
if not container['name'].startswith(cls.RESOURCE_PREFIX):
continue
for obj in cls.swift_region1.get_container(container['name'])[1]:
logging.info('Deleting object {} from {}'.format(
obj['name'],
container['name']))
cls.swift_region1.delete_object(
container['name'],
obj['name'])
logging.info('Deleting container {}'.format(container['name']))
cls.swift_region1.delete_container(container['name'])
def test_two_regions_any_zones_two_replicas(self):
"""Create an object with two replicas across two regions."""
swift_utils.apply_proxy_config(
self.region1_proxy_app,
{
'write-affinity': 'r1, r2',
'write-affinity-node-count': '1',
'replicas': '2'},
self.region1_model_name)
container_name, obj_name, obj_replicas = swift_utils.create_object(
self.swift_region1,
self.region1_proxy_app,
self.storage_topology,
self.RESOURCE_PREFIX,
model_name=self.region1_model_name)
# Check object is accessible from other regions proxy.
self.swift_region2.head_object(container_name, obj_name)
# Check there is at least one replica in each region.
self.assertEqual(
sorted(obj_replicas.distinct_regions),
[1, 2])
# Check there are two relicas
self.assertEqual(
len(obj_replicas.all_zones),
2)
def test_two_regions_any_zones_three_replicas(self):
"""Create an object with three replicas across two regions."""
swift_utils.apply_proxy_config(
self.region1_proxy_app,
{
'write-affinity': 'r1, r2',
'write-affinity-node-count': '1',
'replicas': '3'},
self.region1_model_name)
container_name, obj_name, obj_replicas = swift_utils.create_object(
self.swift_region1,
self.region1_proxy_app,
self.storage_topology,
self.RESOURCE_PREFIX,
model_name=self.region1_model_name)
# Check object is accessible from other regions proxy.
self.swift_region2.head_object(container_name, obj_name)
# Check there is at least one replica in each region.
self.assertEqual(
sorted(obj_replicas.distinct_regions),
[1, 2])
# Check there are three relicas
self.assertEqual(
len(obj_replicas.all_zones),
3)

295
zaza/openstack/utilities/swift.py Normal file
View File

@@ -0,0 +1,295 @@
# Copyright 2019 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Swift utilities."""
import logging
import uuid
import zaza.model
import zaza.openstack.utilities.juju as juju_utils
class ObjectReplica:
"""A replica of an object.
The replica attributes show the location of an object replica.
server: IP address or hostname of machine hosting replica
port: Port of swift object server running on machine hosting replica
device: Path to device hosting replica
handoff_device: Whether this is a handoff devices. Handoff devices pass
the replica on to a remote storage node.
"""
def __init__(self, raw_line):
"""Extract storage info from text."""
rl = raw_line.split()
self.server, self.port = rl[2].split(':')
self.device = rl[3]
self.handoff_device = rl[-1] == '[Handoff]'
class ObjectReplicas:
"""Replicas of an object."""
def __init__(self, proxy_app, account, container_name, object_name,
storage_topology, model_name=None):
"""Find all replicas of given object.
:param proxy_app: Name of proxy application
:type proxy_app: str
:param account: Account that owns the container.
:type account: str
:param container_name: Name of container that contains the object.
:type container_name: str
:param object_name: Name of object.
:type object_name: str
:param storage_topology: Dictionary keyed on IP of storage node info.
:type storage_topology: {}
:param model_name: Model to point environment at
:type model_name: str
"""
self.replicas = []
self.replica_placements = {}
self.storage_topology = storage_topology
raw_output = self.run_get_nodes(
proxy_app,
account,
container_name,
object_name,
model_name=model_name)
for line in self.extract_storage_lines(raw_output):
self.add_replica(line)
def add_replica(self, storage_line):
"""Add a replica to the replica set."""
self.replicas.append(ObjectReplica(storage_line))
def extract_storage_lines(self, raw_output):
"""Extract replica list from output of swift-get-nodes.
:param storage_line: Output of swift-get-nodes
:type storage_line: str
:returns: List of lines relating to replicas.
:rtype: [str, ...]
"""
storage_lines = []
for line in raw_output.split('\n'):
if line.startswith('Server:Port '):
storage_lines.append(line)
return storage_lines
def run_get_nodes(self, proxy_app, account, container_name, object_name,
model_name=None):
"""Run swift-get-nodes for an object on a proxy unit.
:param proxy_app: Name of proxy application
:type proxy_app: str
:param account: Account that owns the container.
:type account: str
:param container_name: Name of container that contains the object.
:type container_name: str
:param object_name: Name of object.
:type object_name: str
:param model_name: Model to point environment at
:type model_name: str
"""
ring_file = '/etc/swift/object.ring.gz'
obj_cmd = "swift-get-nodes -a {} {} {} {}".format(
ring_file,
account,
container_name,
object_name)
cmd_result = zaza.model.run_on_leader(
proxy_app,
obj_cmd,
model_name=model_name)
return cmd_result['Stdout']
@property
def hand_off_ips(self):
"""Replicas which are marked as handoff devices.
These are not real replicas. They hand off the replica to other node.
:returns: List of IPS of handoff nodes for object.
:rtype: [str, ...]
"""
return [r.server for r in self.replicas if r.handoff_device]
@property
def storage_ips(self):
"""Ip addresses of nodes that are housing a replica.
:returns: List of IPS of storage nodes holding a replica of the object.
:rtype: [str, ...]
"""
return [r.server for r in self.replicas if not r.handoff_device]
@property
def placements(self):
"""Region an zone information for each replica.
:returns: List of dicts with region and zone information.
:rtype: [{
'app_name': str,
'unit': juju.Unit,
'region': int,
'zone': int}, ...]
"""
return [self.storage_topology[ip] for ip in self.storage_ips]
@property
def distinct_regions(self):
"""List of distinct regions that have a replica.
:returns: List of regions that have a replica
:rtype: [int, ...]
"""
return list(set([p['region'] for p in self.placements]))
@property
def all_zones(self):
"""List of all zones that have a replica.
:returns: List of tuples (region, zone) that have a replica.
:rtype: [(r1, z1), ...]
"""
return [(p['region'], p['zone']) for p in self.placements]
@property
def distinct_zones(self):
"""List of distinct region + zones that have a replica.
:returns: List of tuples (region, zone) that have a replica.
:rtype: [(r1, z1), ...]
"""
return list(set(self.all_zones))
def get_swift_storage_topology(model_name=None):
"""Get details of storage nodes and which region and zones they belong in.
:param model_name: Model to point environment at
:type model_name: str
:returns: Dictionary of storage nodes and their region/zone information.
:rtype: {
'ip (str)': {
'app_name': str,
'unit': juju.Unit
'region': int,
'zone': int},
...}
"""
topology = {}
status = juju_utils.get_full_juju_status(model_name=model_name)
for app_name, app_dep_config in status.applications.items():
if 'swift-storage' in app_dep_config['charm']:
app_config = zaza.model.get_application_config(
app_name,
model_name=model_name)
region = app_config['region']['value']
zone = app_config['zone']['value']
for unit in zaza.model.get_units(app_name, model_name=model_name):
topology[unit.public_address] = {
'app_name': app_name,
'unit': unit,
'region': region,
'zone': zone}
return topology
def setup_test_container(swift_client, resource_prefix):
"""Create a swift container for use be tests.
:param swift_client: Swift client to use for object creation
:type swift_client: swiftclient.Client
:returns: (container_name, account_name) Container name and account
name for new container
:rtype: (str, str)
"""
run_id = str(uuid.uuid1()).split('-')[0]
container_name = '{}-{}-container'.format(resource_prefix, run_id)
swift_client.put_container(container_name)
resp_headers, containers = swift_client.get_account()
account = resp_headers['x-account-project-domain-id']
return container_name, account
def apply_proxy_config(proxy_app, config, model_name=None):
"""Update the give proxy_app with new charm config.
:param proxy_app: Name of proxy application
:type proxy_app: str
:param config: Dictionary of configuration setting(s) to apply
:type config: dict
:param model_name: Name of model to query.
:type model_name: str
"""
current_config = zaza.model.get_application_config(
proxy_app,
model_name=model_name)
# Although there is no harm in applying config that is a noop it
# does affect the expected behaviour afterwards. So, only apply
# genuine changes so we can safely expect the charm to fire a hook.
for key, value in config.items():
if str(config[key]) != str(current_config[key]['value']):
break
else:
logging.info(
'Config update for {} not required.'.format(proxy_app))
return
logging.info('Updating {} charm settings'.format(proxy_app))
zaza.model.set_application_config(
proxy_app,
config,
model_name=model_name)
zaza.model.block_until_all_units_idle()
def create_object(swift_client, proxy_app, storage_topology, resource_prefix,
model_name=None):
"""Create a test object in a new container.
:param swift_client: Swift client to use for object creation
:type swift_client: swiftclient.Client
:param proxy_app: Name of proxy application
:type proxy_app: str
:param storage_topology: Dictionary keyed on IP of storage node info.
:type storage_topology: {}
:param resource_prefix: Prefix to use when naming new resources
:type resource_prefix: str
:param model_name: Model to point environment at
:type model_name: str
:returns: (container_name, object_name, object replicas)
:rtype: (str, str, ObjectReplicas)
"""
container_name, account = setup_test_container(
swift_client,
resource_prefix)
object_name = 'zaza_test_object.txt'
swift_client.put_object(
container_name,
object_name,
contents='File contents',
content_type='text/plain'
)
obj_replicas = ObjectReplicas(
proxy_app,
account,
container_name,
object_name,
storage_topology,
model_name=model_name)
return container_name, object_name, obj_replicas