#!/usr/bin/env python3 # Copyright 2018 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Collection of tests for vault.""" import logging import tenacity import zaza.model as zaza_model import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.guest as guest_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.utilities.generic as generic_utils from zaza.utilities import juju as juju_utils def _resource_reaches_status( unit, auth_args, status_command, full_status_command, resource_id, target_status ): """Wait for a workload resource to reach a status. :param unit: unit to run cli commands on :type unit: zaza_model.Unit :param auth_args: authentication arguments for command :type auth_args: str :param status_command: command to execute to get the resource status that is expected to reach target_status :type status_command: str :param full_status_command: command to execute to get insights on why the resource failed to reach target_status :type full_status_command: str :param resource_id: resource ID to monitor :type resource_id: str :param target_status: status to monitor for :type target_status: str """ resource_status = ( juju_utils.remote_run( unit, remote_cmd=status_command.format( auth_args=auth_args, resource_id=resource_id ), timeout=180, fatal=True, ) .strip() .split("\n")[-1] ) logging.info( "Checking resource ({}) status: {}".format( resource_id, resource_status ) ) if resource_status == target_status: return full_resource_status = ( juju_utils.remote_run( unit, remote_cmd=full_status_command.format( auth_args=auth_args, resource_id=resource_id ), timeout=180, fatal=True, ) .strip() ) raise Exception("Resource not ready:\n{}".format(full_resource_status)) class WorkloadmgrCLIHelper(object): """Helper for working with workloadmgrcli.""" WORKLOAD_CREATE_CMD = ( "openstack {auth_args} workload create " "--instance instance-id={instance_id} " "-f value -c ID" ) WORKLOAD_STATUS_CMD = ( "openstack {auth_args} workload show " "-f value -c status " "{resource_id}" ) WORKLOAD_FULL_STATUS_CMD = ( "openstack {auth_args} workload show " "{resource_id}" ) SNAPSHOT_CMD = ( "openstack {auth_args} workload snapshot --full {workload_id}" ) SNAPSHOT_ID_CMD = ( "openstack {auth_args} workload snapshot list " "--workload_id {workload_id} " "-f value -c ID" ) SNAPSHOT_STATUS_CMD = ( "openstack {auth_args} workload snapshot show " "-f value -c status " "{resource_id}" ) SNAPSHOT_FULL_STATUS_CMD = ( "openstack {auth_args} workload snapshot show " "{resource_id}" ) ONECLICK_RESTORE_CMD = ( "openstack {auth_args} workload snapshot oneclick-restore " "{snapshot_id} " ) RESTORE_LIST_CMD = ( "openstack {auth_args} workloadmgr restore list " "--snapshot_id {snapshot_id} " "-f value -c ID" ) RESTORE_STATUS_CMD = ( "openstack {auth_args} workloadmgr restore show " "-f value -c status " "{resource_id}" ) RESTORE_FULL_STATUS_CMD = ( "openstack {auth_args} workloadmgr restore show " "{resource_id}" ) def __init__(self, keystone_client): """Initialise helper. :param keystone_client: keystone client :type keystone_client: keystoneclient.v3 """ self.trilio_wlm_unit = zaza_model.get_first_unit_name( "trilio-wlm" ) self.auth_args = self._auth_arguments(keystone_client) @classmethod def _auth_arguments(cls, keystone_client): """Generate workloadmgrcli arguments for cloud authentication. :returns: string of required cli arguments for authentication :rtype: str """ overcloud_auth = openstack_utils.get_overcloud_auth() overcloud_auth.update( { "OS_DOMAIN_ID": openstack_utils.get_domain_id( keystone_client, domain_name="admin_domain" ), "OS_TENANT_ID": openstack_utils.get_project_id( keystone_client, project_name="admin", domain_name="admin_domain", ), "OS_TENANT_NAME": "admin", } ) _required_keys = [ "OS_AUTH_URL", "OS_USERNAME", "OS_PASSWORD", "OS_REGION_NAME", "OS_DOMAIN_ID", "OS_TENANT_ID", "OS_TENANT_NAME", ] params = [] for os_key in _required_keys: params.append( "--{}={}".format( os_key.lower().replace("_", "-"), overcloud_auth[os_key], ) ) return " ".join(params) def create_workload(self, instance_id): """Create a new workload. :param instance_id: instance ID to create workload from :type instance_id: str :returns: workload ID :rtype: str """ workload_id = juju_utils.remote_run( self.trilio_wlm_unit, remote_cmd=self.WORKLOAD_CREATE_CMD.format( auth_args=self.auth_args, instance_id=instance_id ), timeout=180, fatal=True, ).strip() retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=30), stop=tenacity.stop_after_delay(180), reraise=True, ) retryer( _resource_reaches_status, self.trilio_wlm_unit, self.auth_args, self.WORKLOAD_STATUS_CMD, self.WORKLOAD_FULL_STATUS_CMD, workload_id, "available", ) return workload_id def create_snapshot(self, workload_id): """Create a new snapshot. :param workload_id: workload ID to create snapshot from :type workload_id: str :returns: snapshot ID :rtype: str """ juju_utils.remote_run( self.trilio_wlm_unit, remote_cmd=self.SNAPSHOT_CMD.format( auth_args=self.auth_args, workload_id=workload_id ), timeout=180, fatal=True, ) snapshot_id = juju_utils.remote_run( self.trilio_wlm_unit, remote_cmd=self.SNAPSHOT_ID_CMD.format( auth_args=self.auth_args, workload_id=workload_id ), timeout=180, fatal=True, ).strip() retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=30), stop=tenacity.stop_after_delay(720), reraise=True, ) retryer( _resource_reaches_status, self.trilio_wlm_unit, self.auth_args, self.SNAPSHOT_STATUS_CMD, self.SNAPSHOT_FULL_STATUS_CMD, snapshot_id, "available", ) return snapshot_id def oneclick_restore(self, snapshot_id): """Restore a workload from a snapshot. :param snapshot_id: snapshot ID to restore :type snapshot_id: str """ juju_utils.remote_run( self.trilio_wlm_unit, remote_cmd=self.ONECLICK_RESTORE_CMD.format( auth_args=self.auth_args, snapshot_id=snapshot_id ), timeout=180, fatal=True, ) restore_id = juju_utils.remote_run( self.trilio_wlm_unit, remote_cmd=self.RESTORE_LIST_CMD.format( auth_args=self.auth_args, snapshot_id=snapshot_id ), timeout=180, fatal=True, ).strip() retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=30), stop=tenacity.stop_after_delay(720), reraise=True, ) retryer( _resource_reaches_status, self.trilio_wlm_unit, self.auth_args, self.RESTORE_STATUS_CMD, self.RESTORE_FULL_STATUS_CMD, restore_id, "available", ) return restore_id class TrilioBaseTest(test_utils.OpenStackBaseTest): """Base test class for charms.""" RESOURCE_PREFIX = "zaza-triliovault-tests" conf_file = None @classmethod def setUpClass(cls): """Run class setup for running tests.""" super().setUpClass() cls.cinder_client = openstack_utils.get_cinder_session_client( cls.keystone_session ) cls.nova_client = openstack_utils.get_nova_session_client( cls.keystone_session ) cls.keystone_client = openstack_utils.get_keystone_session_client( cls.keystone_session ) def test_restart_on_config_change(self): """Check restart happens on config change. Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ # Expected default and alternate values set_default = {"debug": False} set_alternate = {"debug": True} # Make config change, check for service restarts self.restart_on_changed( self.conf_file, set_default, set_alternate, {"DEFAULT": {"debug": ["False"]}}, {"DEFAULT": {"debug": ["True"]}}, self.services, ) def test_pause_resume(self): """Run pause and resume tests. Pause service and check services are stopped then resume and check they are started """ with self.pause_resume(self.services, pgrep_full=False): logging.info("Testing pause resume") def test_snapshot_workload(self): """Ensure that a workload can be created and snapshot'ed.""" # Setup volume and instance and attach one to the other volume = openstack_utils.create_volume( self.cinder_client, size="1", name="{}-100-vol".format(self.RESOURCE_PREFIX), ) instance = guest_utils.launch_instance( glance_setup.CIRROS_IMAGE_NAME, vm_name="{}-server".format(self.RESOURCE_PREFIX), ) # Trilio need direct access to ceph - OMG openstack_utils.attach_volume( self.nova_client, volume.id, instance.id ) workloadmgrcli = WorkloadmgrCLIHelper(self.keystone_client) # Create workload using instance logging.info("Creating workload configuration") workload_id = workloadmgrcli.create_workload(instance.id) logging.info("Created workload: {}".format(workload_id)) logging.info("Initiating snapshot") snapshot_id = workloadmgrcli.create_snapshot(workload_id) logging.info( "Snapshot of workload {} created: {}".format( workload_id, snapshot_id ) ) logging.info("Deleting server and volume ready for restore") openstack_utils.delete_resource( self.nova_client.servers, instance.id, "deleting instance" ) # NOTE: Trilio leaves a snapshot in place - # drop before volume deletion. for ( volume_snapshot ) in self.cinder_client.volume_snapshots.list(): openstack_utils.delete_resource( self.cinder_client.volume_snapshots, volume_snapshot.id, "deleting snapshot", ) openstack_utils.delete_resource( self.cinder_client.volumes, volume.id, "deleting volume" ) logging.info("Initiating restore") workloadmgrcli.oneclick_restore(snapshot_id) class TrilioGhostNFSShareTest(TrilioBaseTest): """Tests for Trilio charms providing the ghost-share action.""" def test_ghost_nfs_share(self): """Ensure ghost-share action bind mounts NFS share.""" generic_utils.assertActionRanOK(zaza_model.run_action( self.lead_unit, 'ghost-share', action_params={ 'nfs-shares': '10.20.0.1:/srv/testing' }, model_name=self.model_name) ) class TrilioWLMTest(TrilioGhostNFSShareTest): """Tests for Trilio Workload Manager charm.""" conf_file = "/etc/workloadmgr/workloadmgr.conf" application_name = "trilio-wlm" services = [ "workloadmgr-api", "workloadmgr-scheduler", "workloadmgr-workloads", "workloadmgr-cron", ] class TrilioDMAPITest(TrilioBaseTest): """Tests for Trilio Data Mover API charm.""" conf_file = "/etc/dmapi/dmapi.conf" application_name = "trilio-dm-api" services = ["dmapi-api"] class TrilioDataMoverTest(TrilioGhostNFSShareTest): """Tests for Trilio Data Mover charm.""" conf_file = "/etc/tvault-contego/tvault-contego.conf" application_name = "trilio-data-mover" services = ["tvault-contego"]