From eb0cba9efc0ba5c665d034986423ddee13503aec Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 20 Apr 2020 18:10:03 +0100 Subject: [PATCH] Add zaza tests for TrilioVault Add setup and tests for trilio-{data-mover,dm-api,wlm} charms. Add attach_volume utility to attach cinder volumes to nova servers. --- zaza/openstack/charm_tests/trilio/__init__.py | 15 + zaza/openstack/charm_tests/trilio/setup.py | 83 ++++ zaza/openstack/charm_tests/trilio/tests.py | 379 ++++++++++++++++++ zaza/openstack/utilities/openstack.py | 40 ++ 4 files changed, 517 insertions(+) create mode 100644 zaza/openstack/charm_tests/trilio/__init__.py create mode 100644 zaza/openstack/charm_tests/trilio/setup.py create mode 100644 zaza/openstack/charm_tests/trilio/tests.py diff --git a/zaza/openstack/charm_tests/trilio/__init__.py b/zaza/openstack/charm_tests/trilio/__init__.py new file mode 100644 index 0000000..d22e570 --- /dev/null +++ b/zaza/openstack/charm_tests/trilio/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing TrilioVault.""" diff --git a/zaza/openstack/charm_tests/trilio/setup.py b/zaza/openstack/charm_tests/trilio/setup.py new file mode 100644 index 0000000..a7ba7b3 --- /dev/null +++ b/zaza/openstack/charm_tests/trilio/setup.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring Trilio.""" + +import logging +import os + +import zaza.model as zaza_model +import zaza.openstack.utilities.juju as juju_utils +import zaza.openstack.utilities.generic as generic_utils + + +def basic_setup(): + """Run setup for testing Trilio. + + Setup for testing Trilio is currently part of functional + tests. + """ + logging.info("Configuring NFS Server") + nfs_server_ip = zaza_model.get_app_ips("nfs-server-test-fixture")[0] + trilio_wlm_unit = zaza_model.get_first_unit_name("trilio-wlm") + + nfs_shares_conf = {"nfs-shares": "{}:/srv/testing".format(nfs_server_ip)} + _trilio_services = ["trilio-wlm", "trilio-data-mover"] + + conf_changed = False + for juju_service in _trilio_services: + app_config = zaza_model.get_application_config(juju_service) + if app_config["nfs-shares"] != nfs_shares_conf["nfs-shares"]: + zaza_model.set_application_config(juju_service, nfs_shares_conf) + conf_changed = True + + if conf_changed: + zaza_model.wait_for_agent_status() + # NOTE(jamespage): wlm-api service must be running in order + # to execute the setup actions + zaza_model.block_until_service_status( + unit_name=trilio_wlm_unit, + services=["wlm-api"], + target_status="active", + ) + + logging.info("Executing create-cloud-admin-trust") + password = juju_utils.leader_get("keystone", "admin_passwd") + + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "trilio-wlm", + "create-cloud-admin-trust", + raise_on_failure=True, + action_params={"password": password}, + ) + ) + + logging.info("Executing create-license") + test_license = os.environ.get("TEST_TRILIO_LICENSE") + if test_license and os.path.exists(test_license): + zaza_model.attach_resource("trilio-wlm", + resource_name='license', + resource_path=test_license) + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "trilio-wlm", "create-license", + raise_on_failure=True + ) + ) + + else: + logging.error("Unable to find Trilio License file") diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py new file mode 100644 index 0000000..e7595a5 --- /dev/null +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of tests for vault.""" + +import logging +import tenacity + +import zaza.model as zaza_model + +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.juju as juju_utils +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.configure.guest as guest_utils + + +def _resource_reaches_status( + unit, auth_args, command, resource_id, target_status +): + """Wait for a workload resource to reach a status. + + :param unit: unit to run cli commands on + :type unit: zaza_model.Unit + :param auth_args: authentication arguments for command + :type auth_args: str + :param command: command to execute + :type command: str + :param resource_id: resource ID to monitor + :type resource_id: str + :param target_status: status to monitor for + :type target_status: str + """ + resource_status = ( + juju_utils.remote_run( + unit, + remote_cmd=command.format( + auth_args=auth_args, resource_id=resource_id + ), + timeout=180, + fatal=True, + ) + .strip() + .split("\n")[-1] + ) + logging.info( + "Checking resource ({}) status: {}".format( + resource_id, resource_status + ) + ) + if resource_status == target_status: + return + raise Exception("Resource not ready: {}".format(resource_status)) + + +class WorkloadmgrCLIHelper(object): + """Helper for working with workloadmgrcli.""" + + WORKLOAD_CREATE_CMD = ( + "openstack {auth_args} workload create " + "--instance instance-id={instance_id} " + "-f value -c ID" + ) + + WORKLOAD_STATUS_CMD = ( + "openstack {auth_args} workload show " + "-f value -c status " + " {resource_id} " + ) + + SNAPSHOT_CMD = ( + "openstack {auth_args} workload snapshot --full {workload_id}" + ) + + SNAPSHOT_ID_CMD = ( + "openstack {auth_args} workload snapshot list " + "--workload_id {workload_id} " + "-f value -c ID" + ) + + SNAPSHOT_STATUS_CMD = ( + "openstack {auth_args} workload snapshot show " + "-f value -c status " + "{resource_id} " + ) + + ONECLICK_RESTORE_CMD = ( + "openstack {auth_args} workload snapshot oneclick-restore " + "{snapshot_id} " + ) + + def __init__(self, keystone_client): + """Initialise helper. + + :param keystone_client: keystone client + :type keystone_client: keystoneclient.v3 + """ + self.trilio_wlm_unit = zaza_model.get_first_unit_name("trilio-wlm") + self.auth_args = self._auth_arguments(keystone_client) + + @classmethod + def _auth_arguments(cls, keystone_client): + """Generate workloadmgrcli arguments for cloud authentication. + + :returns: string of required cli arguments for authentication + :rtype: str + """ + overcloud_auth = openstack_utils.get_overcloud_auth() + overcloud_auth.update( + { + "OS_DOMAIN_ID": openstack_utils.get_domain_id( + keystone_client, domain_name="admin_domain" + ), + "OS_TENANT_ID": openstack_utils.get_project_id( + keystone_client, + project_name="admin", + domain_name="admin_domain", + ), + "OS_TENANT_NAME": "admin", + } + ) + + _required_keys = [ + "OS_AUTH_URL", + "OS_USERNAME", + "OS_PASSWORD", + "OS_REGION_NAME", + "OS_DOMAIN_ID", + "OS_TENANT_ID", + "OS_TENANT_NAME", + ] + + params = [] + for os_key in _required_keys: + params.append( + "--{}={}".format( + os_key.lower().replace("_", "-"), overcloud_auth[os_key] + ) + ) + return " ".join(params) + + def create_workload(self, instance_id): + """Create a new workload. + + :param instance_id: instance ID to create workload from + :type instance_id: str + :returns: workload ID + :rtype: str + """ + workload_id = juju_utils.remote_run( + self.trilio_wlm_unit, + remote_cmd=self.WORKLOAD_CREATE_CMD.format( + auth_args=self.auth_args, instance_id=instance_id + ), + timeout=180, + fatal=True, + ).strip() + + retryer = tenacity.Retrying( + wait=tenacity.wait_exponential(multiplier=1, max=60), + stop=tenacity.stop_after_delay(180), + reraise=True, + ) + retryer( + _resource_reaches_status, + self.trilio_wlm_unit, + self.auth_args, + self.WORKLOAD_STATUS_CMD, + workload_id, + "available", + ) + + return workload_id + + def create_snapshot(self, workload_id): + """Create a new snapshot. + + :param workload_id: workload ID to create snapshot from + :type workload_id: str + :returns: snapshot ID + :rtype: str + """ + juju_utils.remote_run( + self.trilio_wlm_unit, + remote_cmd=self.SNAPSHOT_CMD.format( + auth_args=self.auth_args, workload_id=workload_id + ), + timeout=180, + fatal=True, + ) + snapshot_id = juju_utils.remote_run( + self.trilio_wlm_unit, + remote_cmd=self.SNAPSHOT_ID_CMD.format( + auth_args=self.auth_args, workload_id=workload_id + ), + timeout=180, + fatal=True, + ).strip() + + retryer = tenacity.Retrying( + wait=tenacity.wait_exponential(multiplier=1, max=60), + stop=tenacity.stop_after_delay(720), + reraise=True, + ) + + retryer( + _resource_reaches_status, + self.trilio_wlm_unit, + self.auth_args, + self.SNAPSHOT_STATUS_CMD, + snapshot_id, + "available", + ) + + return snapshot_id + + def oneclick_restore(self, snapshot_id): + """Restore a workload from a snapshot. + + :param snapshot_id: snapshot ID to restore + :type snapshot_id: str + """ + juju_utils.remote_run( + self.trilio_wlm_unit, + remote_cmd=self.ONECLICK_RESTORE_CMD.format( + auth_args=self.auth_args, snapshot_id=snapshot_id + ), + timeout=180, + fatal=True, + ) + + # TODO validate restore but currently failing with 4.0 + # pre-release + + +class TrilioBaseTest(test_utils.OpenStackBaseTest): + """Base test class for charms.""" + + RESOURCE_PREFIX = "zaza-triliovault-tests" + conf_file = None + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super().setUpClass() + cls.cinder_client = openstack_utils.get_cinder_session_client( + cls.keystone_session + ) + cls.nova_client = openstack_utils.get_nova_session_client( + cls.keystone_session + ) + cls.keystone_client = openstack_utils.get_keystone_session_client( + cls.keystone_session + ) + + def test_restart_on_config_change(self): + """Check restart happens on config change. + + Change debug mode and assert that change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + set_default = {"debug": False} + set_alternate = {"debug": True} + + # Make config change, check for service restarts + self.restart_on_changed( + self.conf_file, + set_default, + set_alternate, + {"DEFAULT": {"debug": ["False"]}}, + {"DEFAULT": {"debug": ["True"]}}, + self.services, + ) + + def test_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + with self.pause_resume(self.services, pgrep_full=False): + logging.info("Testing pause resume") + + def test_snapshot_workload(self): + """Ensure that a workload can be created and snapshot'ed.""" + # Setup volume and instance and attach one to the other + volume = openstack_utils.create_volume( + self.cinder_client, + size="1", + name="{}-100-vol".format(self.RESOURCE_PREFIX), + ) + + instance = guest_utils.launch_instance( + glance_setup.CIRROS_IMAGE_NAME, + vm_name="{}-server".format(self.RESOURCE_PREFIX), + ) + + # Trilio need direct access to ceph - OMG + openstack_utils.attach_volume(self.nova_client, volume.id, instance.id) + + workloadmgrcli = WorkloadmgrCLIHelper(self.keystone_client) + + # Create workload using instance + logging.info("Creating workload configuration") + workload_id = workloadmgrcli.create_workload(instance.id) + logging.info("Created workload: {}".format(workload_id)) + + logging.info("Initiating snapshot") + snapshot_id = workloadmgrcli.create_snapshot(workload_id) + logging.info( + "Snapshot of workload {} created: {}".format( + workload_id, snapshot_id + ) + ) + + logging.info("Deleting server and volume ready for restore") + openstack_utils.delete_resource( + self.nova_client.servers, instance.id, "deleting instance" + ) + # NOTE: Trilio leaves a snapshot in place - + # drop before volume deletion. + for volume_snapshot in self.cinder_client.volume_snapshots.list(): + openstack_utils.delete_resource( + self.cinder_client.volume_snapshots, + volume_snapshot.id, + "deleting snapshot", + ) + openstack_utils.delete_resource( + self.cinder_client.volumes, volume.id, "deleting volume" + ) + + logging.info("Initiating restore") + workloadmgrcli.oneclick_restore(snapshot_id) + + +class TrilioWLMTest(TrilioBaseTest): + """Tests for Trilio Workload Manager charm.""" + + conf_file = "/etc/workloadmgr/workloadmgr.conf" + application_name = "trilio-wlm" + + services = [ + "workloadmgr-api", + "workloadmgr-scheduler", + "workloadmgr-workloads", + "workloadmgr-cron", + ] + + +class TrilioDMAPITest(TrilioBaseTest): + """Tests for Trilio Data Mover API charm.""" + + conf_file = "/etc/dmapi/dmapi.conf" + application_name = "trilio-dm-api" + + services = ["dmapi-api"] + + +class TrilioDataMoverTest(TrilioBaseTest): + """Tests for Trilio Data Mover charm.""" + + conf_file = "/etc/tvault-contego/tvault-contego.conf" + application_name = "trilio-data-mover" + + services = ["tvault-contego"] diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 875071b..3e72faf 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -487,6 +487,24 @@ def get_project_id(ks_client, project_name, api_version=2, domain_name=None): return None +def get_domain_id(ks_client, domain_name, api_version=2): + """Return domain ID. + + :param ks_client: Authenticated keystoneclient + :type ks_client: keystoneclient.v3.Client object + :param domain_name: Name of the domain + :type domain_name: string + :param api_version: API version number + :type api_version: int + :returns: Domain ID + :rtype: string or None + """ + all_domains = ks_client.domains.list(name=domain_name) + if all_domains: + return all_domains[0].id + return None + + # Neutron Helpers def get_gateway_uuids(): """Return machine uuids for neutron-gateway(s). @@ -2069,6 +2087,28 @@ def create_volume(cinder, size, name=None, image=None): return volume +def attach_volume(nova, volume_id, instance_id): + """Attach a cinder volume to a nova instance. + + :param nova: Authenticated nova client + :type nova: novaclient.v2.client.Client + :param volume_id: the id of the volume to attach + :type volume_id: str + :param instance_id: the id of the instance to attach the volume to + :type instance_id: str + :returns: nova volume pointer + :rtype: novaclient.v2.volumes.Volume + """ + logging.info( + 'Attaching volume {} to instance {}'.format( + volume_id, instance_id + ) + ) + return nova.volumes.create_server_volume(server_id=instance_id, + volume_id=volume_id, + device='/dev/vdx') + + def create_volume_backup(cinder, volume_id, name=None): """Create cinder volume backup.