diff --git a/setup.py b/setup.py index e1689f3..d1c6970 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,10 @@ install_require = [ 'juju-wait', 'PyYAML', 'tenacity', + 'oslo.config', + 'python-keystoneclient', + 'python-novaclient', + 'python-neutronclient', ] tests_require = [ diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 016fd17..75d657a 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1,5 +1,7 @@ import copy import mock +import tenacity + import unit_tests.utils as ut_utils from zaza.utilities import openstack as openstack_utils @@ -171,3 +173,157 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): openstack_utils.get_undercloud_keystone_session() self.get_keystone_session.assert_called_once_with(_auth, scope=_scope) + + def test_get_urllib_opener(self): + self.patch_object(openstack_utils.urllib.request, "ProxyHandler") + self.patch_object(openstack_utils.urllib.request, "HTTPHandler") + self.patch_object(openstack_utils.urllib.request, "build_opener") + self.patch_object(openstack_utils.os, "getenv") + self.getenv.return_value = None + HTTPHandler_mock = mock.MagicMock() + self.HTTPHandler.return_value = HTTPHandler_mock + openstack_utils.get_urllib_opener() + self.build_opener.assert_called_once_with(HTTPHandler_mock) + self.HTTPHandler.assert_called_once_with() + + def test_get_urllib_opener_proxy(self): + self.patch_object(openstack_utils.urllib.request, "ProxyHandler") + self.patch_object(openstack_utils.urllib.request, "HTTPHandler") + self.patch_object(openstack_utils.urllib.request, "build_opener") + self.patch_object(openstack_utils.os, "getenv") + self.getenv.return_value = 'http://squidy' + ProxyHandler_mock = mock.MagicMock() + self.ProxyHandler.return_value = ProxyHandler_mock + openstack_utils.get_urllib_opener() + self.build_opener.assert_called_once_with(ProxyHandler_mock) + self.ProxyHandler.assert_called_once_with({'http': 'http://squidy'}) + + def test_find_cirros_image(self): + urllib_opener_mock = mock.MagicMock() + self.patch_object(openstack_utils, "get_urllib_opener") + self.get_urllib_opener.return_value = urllib_opener_mock + urllib_opener_mock.open().read.return_value = b'12' + self.assertEqual( + openstack_utils.find_cirros_image('aarch64'), + 'http://download.cirros-cloud.net/12/cirros-12-aarch64-disk.img') + + def test_download_image(self): + urllib_opener_mock = mock.MagicMock() + self.patch_object(openstack_utils, "get_urllib_opener") + self.get_urllib_opener.return_value = urllib_opener_mock + self.patch_object(openstack_utils.urllib.request, "install_opener") + self.patch_object(openstack_utils.urllib.request, "urlretrieve") + openstack_utils.download_image('http://cirros/c.img', '/tmp/c1.img') + self.install_opener.assert_called_once_with(urllib_opener_mock) + self.urlretrieve.assert_called_once_with( + 'http://cirros/c.img', '/tmp/c1.img') + + def test_resource_reaches_status(self): + resource_mock = mock.MagicMock() + resource_mock.get.return_value = mock.MagicMock(status='available') + openstack_utils.resource_reaches_status(resource_mock, 'e01df65a') + + def test_resource_reaches_status_fail(self): + openstack_utils.resource_reaches_status.retry.wait = \ + tenacity.wait_none() + resource_mock = mock.MagicMock() + resource_mock.get.return_value = mock.MagicMock(status='unavailable') + with self.assertRaises(AssertionError): + openstack_utils.resource_reaches_status( + resource_mock, + 'e01df65a') + + def test_resource_reaches_status_bespoke(self): + resource_mock = mock.MagicMock() + resource_mock.get.return_value = mock.MagicMock(status='readyish') + openstack_utils.resource_reaches_status( + resource_mock, + 'e01df65a', + 'readyish') + + def test_resource_reaches_status_bespoke_fail(self): + openstack_utils.resource_reaches_status.retry.wait = \ + tenacity.wait_none() + resource_mock = mock.MagicMock() + resource_mock.get.return_value = mock.MagicMock(status='available') + with self.assertRaises(AssertionError): + openstack_utils.resource_reaches_status( + resource_mock, + 'e01df65a', + 'readyish') + + def test_resource_removed(self): + resource_mock = mock.MagicMock() + resource_mock.list.return_value = [mock.MagicMock(id='ba8204b0')] + openstack_utils.resource_removed(resource_mock, 'e01df65a') + + def test_resource_removed_fail(self): + openstack_utils.resource_reaches_status.retry.wait = \ + tenacity.wait_none() + resource_mock = mock.MagicMock() + resource_mock.list.return_value = [mock.MagicMock(id='e01df65a')] + with self.assertRaises(AssertionError): + openstack_utils.resource_removed(resource_mock, 'e01df65a') + + def test_delete_resource(self): + resource_mock = mock.MagicMock() + self.patch_object(openstack_utils, "resource_removed") + openstack_utils.delete_resource(resource_mock, 'e01df65a') + resource_mock.delete.assert_called_once_with('e01df65a') + self.resource_removed.assert_called_once_with( + resource_mock, + 'e01df65a', + 'resource') + + def test_delete_image(self): + self.patch_object(openstack_utils, "delete_resource") + glance_mock = mock.MagicMock() + openstack_utils.delete_image(glance_mock, 'b46c2d83') + self.delete_resource.assert_called_once_with( + glance_mock.images, + 'b46c2d83', + msg="glance image") + + def test_upload_image_to_glance(self): + self.patch_object(openstack_utils, "resource_reaches_status") + glance_mock = mock.MagicMock() + image_mock = mock.MagicMock(id='9d1125af') + glance_mock.images.create.return_value = image_mock + m = mock.mock_open() + with mock.patch('zaza.utilities.openstack.open', m, create=False) as f: + openstack_utils.upload_image_to_glance( + glance_mock, + '/tmp/im1.img', + 'bob') + glance_mock.images.create.assert_called_once_with( + name='bob', + disk_format='qcow2', + visibility='public', + container_format='bare') + glance_mock.images.upload.assert_called_once_with( + '9d1125af', + f(), + ) + self.resource_reaches_status.assert_called_once_with( + glance_mock.images, + '9d1125af', + expected_stat='active', + msg='Image status wait') + + def test_create_image(self): + glance_mock = mock.MagicMock() + self.patch_object(openstack_utils.os.path, "exists") + self.patch_object(openstack_utils, "download_image") + self.patch_object(openstack_utils, "upload_image_to_glance") + openstack_utils.create_image( + glance_mock, + 'http://cirros/c.img', + 'bob') + self.exists.return_value = False + self.download_image.assert_called_once_with( + 'http://cirros/c.img', + 'tests/c.img') + self.upload_image_to_glance.assert_called_once_with( + glance_mock, + 'tests/c.img', + 'bob') diff --git a/zaza/charm_tests/glance/__init__.py b/zaza/charm_tests/glance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/charm_tests/glance/setup.py b/zaza/charm_tests/glance/setup.py new file mode 100644 index 0000000..a7f0819 --- /dev/null +++ b/zaza/charm_tests/glance/setup.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + + +def basic_setup(): + """Glance setup for testing glance is currently part of glance functional + tests. Image setup for other tests to use should go here""" + pass diff --git a/zaza/charm_tests/glance/tests.py b/zaza/charm_tests/glance/tests.py new file mode 100644 index 0000000..9bd9709 --- /dev/null +++ b/zaza/charm_tests/glance/tests.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +import logging + +import zaza.utilities.openstack as openstack_utils +import zaza.charm_tests.test_utils as test_utils + + +class GlanceTest(test_utils.OpenStackAPITest): + + @classmethod + def setUpClass(cls): + super(GlanceTest, cls).setUpClass() + cls.glance_client = openstack_utils.get_glance_session_client( + cls.keystone_session) + + def test_410_glance_image_create_delete(self): + """Create an image and then delete it""" + image_url = openstack_utils.find_cirros_image(arch='x86_64') + image = openstack_utils.create_image( + self.glance_client, + image_url, + 'cirrosimage') + openstack_utils.delete_image(self.glance_client, image.id) + + def test_411_set_disk_format(self): + """Change disk format and assert then change propagates to the correct + file and that services are restarted as a result""" + # Expected default and alternate values + set_default = { + 'disk-formats': 'ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar'} + set_alternate = {'disk-formats': 'qcow2'} + + # Config file affected by juju set config change + conf_file = '/etc/glance/glance-api.conf' + + # Make config change, check for service restarts + logging.debug('Setting disk format glance...') + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + {'image_format': { + 'disk_formats': [ + 'ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar']}}, + {'image_format': {'disk_formats': ['qcow2']}}, + ['glance-api']) + + def test_901_pause_resume(self): + """Pause service and check services are stopped then resume and check + they are started""" + self.pause_resume(['glance-api']) diff --git a/zaza/charm_tests/test_utils.py b/zaza/charm_tests/test_utils.py index 81bbf6d..3060c26 100644 --- a/zaza/charm_tests/test_utils.py +++ b/zaza/charm_tests/test_utils.py @@ -1,13 +1,18 @@ import logging +import unittest import zaza.model -import zaza.charm_lifecycle.utils as utils +import zaza.model as model +import zaza.charm_lifecycle.utils as lifecycle_utils +import zaza.utilities.openstack as openstack_utils def skipIfNotHA(service_name): def _skipIfNotHA_inner_1(f): def _skipIfNotHA_inner_2(*args, **kwargs): - ips = zaza.model.get_app_ips(utils.get_juju_model(), service_name) + ips = zaza.model.get_app_ips( + lifecycle_utils.get_juju_model(), + service_name) if len(ips) > 1: return f(*args, **kwargs) else: @@ -16,3 +21,136 @@ def skipIfNotHA(service_name): return _skipIfNotHA_inner_2 return _skipIfNotHA_inner_1 + + +class OpenStackAPITest(unittest.TestCase): + """Generic helpers for testing OpenStack API charms""" + + @classmethod + def setUpClass(cls): + cls.keystone_session = openstack_utils.get_overcloud_keystone_session() + cls.model_name = lifecycle_utils.get_juju_model() + cls.test_config = lifecycle_utils.get_charm_config() + cls.application_name = cls.test_config['charm_name'] + + def restart_on_changed(self, config_file, default_config, alternate_config, + default_entry, alternate_entry, services): + """Test that changing config results in config file being updates and + services restarted. Return config to default_config afterwards + + :param config_file: Config file to check for settings + :type config_file: str + :param default_config: Dict of charm settings to set on completion + :type default_config: dict + :param alternate_config: Dict of charm settings to change to + :type alternate_config: dict + :param default_entry: Config file entries that correspond to + default_config + :type default_entry: dict + :param alternate_entry: Config file entries that correspond to + :type alternate_entry: alternate_config + :param services: Services expected to be restarted when config_file is + changed. + :type services: list + """ + # first_unit is only useed to grab a timestamp, the assumption being + # that all the units times are in sync. + first_unit = model.get_first_unit_name( + self.model_name, + self.application_name) + logging.debug('First unit is {}'.format(first_unit)) + + mtime = model.get_unit_time(self.model_name, first_unit) + logging.debug('Remote unit timestamp {}'.format(mtime)) + + logging.debug('Changing charm setting to {}'.format(alternate_config)) + model.set_application_config( + self.model_name, + self.application_name, + alternate_config) + + logging.debug( + 'Waiting for updates to propagate to {}'.format(config_file)) + model.block_until_oslo_config_entries_match( + self.model_name, + self.application_name, + config_file, + alternate_entry) + + logging.debug( + 'Waiting for units to reach target states'.format(config_file)) + model.wait_for_application_states( + self.model_name, + self.test_config.get('target_deploy_status', {})) + + # Config update has occured and hooks are idle. Any services should + # have been restarted by now: + logging.debug( + 'Waiting for services ({}) to be restarted'.format(services)) + model.block_until_services_restarted( + self.model_name, + self.application_name, + mtime, + services) + + logging.debug('Restoring charm setting to {}'.format(default_config)) + model.set_application_config( + self.model_name, + self.application_name, + default_config) + + logging.debug( + 'Waiting for updates to propagate to '.format(config_file)) + model.block_until_oslo_config_entries_match( + self.model_name, + self.application_name, + config_file, + default_entry) + + logging.debug( + 'Waiting for units to reach target states'.format(config_file)) + model.wait_for_application_states( + self.model_name, + self.test_config.get('target_deploy_status', {})) + + def pause_resume(self, services): + """Pause and then resume a unit checking that services are in the + required state after each action + + :param services: Services expected to be restarted when config_file is + changed. + :type services: list + """ + first_unit = model.get_first_unit_name( + self.model_name, + self.application_name) + model.block_until_service_status( + self.model_name, + first_unit, + services, + 'running') + model.block_until_unit_wl_status( + self.model_name, + first_unit, + 'active') + model.run_action(self.model_name, first_unit, 'pause', {}) + model.block_until_unit_wl_status( + self.model_name, + first_unit, + 'maintenance') + model.block_until_all_units_idle(self.model_name) + model.block_until_service_status( + self.model_name, + first_unit, + services, + 'stopped') + model.run_action(self.model_name, first_unit, 'resume', {}) + model.block_until_unit_wl_status( + self.model_name, + first_unit, + 'active') + model.block_until_all_units_idle(self.model_name) + model.block_until_service_status( + self.model_name, + first_unit, services, + 'running') diff --git a/zaza/utilities/openstack.py b/zaza/utilities/openstack.py index 696b74b..090bf61 100644 --- a/zaza/utilities/openstack.py +++ b/zaza/utilities/openstack.py @@ -1,11 +1,11 @@ -#!/usr/bin/env python - from .os_versions import ( OPENSTACK_CODENAMES, SWIFT_CODENAMES, PACKAGE_CODENAMES, ) +from glanceclient import Client as GlanceClient + from keystoneclient.v3 import client as keystoneclient_v3 from keystoneauth1 import session from keystoneauth1.identity import ( @@ -16,12 +16,14 @@ from novaclient import client as novaclient_client from neutronclient.v2_0 import client as neutronclient from neutronclient.common import exceptions as neutronexceptions +import juju_wait import logging import os import re import six import sys -import juju_wait +import tenacity +import urllib from zaza import model from zaza.charm_lifecycle import utils as lifecycle_utils @@ -31,6 +33,9 @@ from zaza.utilities import ( juju as juju_utils, ) +CIRROS_RELEASE_URL = 'http://download.cirros-cloud.net/version/released' +CIRROS_IMAGE_URL = 'http://download.cirros-cloud.net' + CHARM_TYPES = { 'neutron': { 'pkg': 'neutron-common', @@ -116,6 +121,17 @@ def get_ks_creds(cloud_creds, scope='PROJECT'): return auth +def get_glance_session_client(session): + """Return glanceclient authenticated by keystone session + + :param session: Keystone session object + :type session: keystoneauth1.session.Session object + :returns: Authenticated glanceclient + :rtype: glanceclient.Client + """ + return GlanceClient('2', session=session) + + def get_nova_session_client(session): """Return novaclient authenticated by keystone session @@ -1269,3 +1285,179 @@ def get_overcloud_auth(): 'API_VERSION': 3, } return auth_settings + + +def get_urllib_opener(): + """Create a urllib opener taking into account proxy settings + + Using urllib.request.urlopen will automatically handle proxies so none + of this function is needed except we are currently specifying proxies + via AMULET_HTTP_PROXY rather than http_proxy so a ProxyHandler is needed + explicitly stating the proxies. + + :returns: An opener which opens URLs via BaseHandlers chained together + :rtype: urllib.request.OpenerDirector + """ + http_proxy = os.getenv('AMULET_HTTP_PROXY') + logging.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + + if http_proxy: + handler = urllib.request.ProxyHandler({'http': http_proxy}) + else: + handler = urllib.request.HTTPHandler() + return urllib.request.build_opener(handler) + + +def find_cirros_image(arch): + """Return the url for the latest cirros image for the given architecture + + :param arch: aarch64, arm, i386, x86_64 etc + :type arch: str + :returns: Unit matching given name + :rtype: juju.unit.Unit or None + """ + opener = get_urllib_opener() + f = opener.open(CIRROS_RELEASE_URL) + version = f.read().strip().decode() + cirros_img = 'cirros-{}-{}-disk.img'.format(version, arch) + return '{}/{}/{}'.format(CIRROS_IMAGE_URL, version, cirros_img) + + +def download_image(image_url, target_file): + """Download the image from the given url to the specified file + + :param image_url: URL to download image from + :type image_url: str + :param target_file: Local file to savee image to + :type target_file: str + """ + opener = get_urllib_opener() + urllib.request.install_opener(opener) + urllib.request.urlretrieve(image_url, target_file) + + +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), + reraise=True, stop=tenacity.stop_after_attempt(8)) +def resource_reaches_status(resource, resource_id, + expected_stat='available', + msg='resource'): + """Wait for an openstack resources status to reach an expected status + within a specified time. Useful to confirm that nova instances, cinder + vols, snapshots, glance images, heat stacks and other resources + eventually reach the expected status. + + :param resource: pointer to os resource type, ex: heat_client.stacks + :type resource: str + :param resource_id: unique id for the openstack resource + :type resource_id: str + :param expected_stat: status to expect resource to reach + :type expected_stat: str + :param msg: text to identify purpose in logging + :type msy: str + :raises: AssertionError + """ + resource_stat = resource.get(resource_id).status + assert resource_stat == expected_stat, ( + "Resource in {} state, waiting for {}" .format(resource_stat, + expected_stat,)) + + +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), + reraise=True, stop=tenacity.stop_after_attempt(2)) +def resource_removed(resource, resource_id, msg="resource"): + """Wait for an openstack resource to no longer be present + + :param resource: pointer to os resource type, ex: heat_client.stacks + :type resource: str + :param resource_id: unique id for the openstack resource + :type resource_id: str + :param msg: text to identify purpose in logging + :type msy: str + :raises: AssertionError + """ + matching = [r for r in resource.list() if r.id == resource_id] + logging.debug("Resource {} still present".format(resource_id)) + assert len(matching) == 0, "Resource {} still present".format(resource_id) + + +def delete_resource(resource, resource_id, msg="resource"): + """Delete an openstack resource, such as one instance, keypair, + image, volume, stack, etc., and confirm deletion within max wait time. + + :param resource: pointer to os resource type, ex:glance_client.images + :type resource: str + :param resource_id: unique name or id for the openstack resource + :type resource_id: str + :param msg: text to identify purpose in logging + :type msg: str + """ + logging.debug('Deleting OpenStack resource ' + '{} ({})'.format(resource_id, msg)) + resource.delete(resource_id) + resource_removed(resource, resource_id, msg) + + +def delete_image(glance, img_id): + """Delete the given image from glance + + :param glance: Authenticated glanceclient + :type glance: glanceclient.Client + :param img_id: unique name or id for the openstack resource + :type img_id: str + """ + delete_resource(glance.images, img_id, msg="glance image") + + +def upload_image_to_glance(glance, local_path, image_name): + """Upload the given image to glance and apply the given label + + :param glance: Authenticated glanceclient + :type glance: glanceclient.Client + :param local_path: Path to local image + :type local_path: str + :param image_name: The label to give the image in glance + :type image_name: str + """ + # Create glance image + image = glance.images.create( + name=image_name, + disk_format='qcow2', + visibility='public', + container_format='bare') + glance.images.upload(image.id, open(local_path, 'rb')) + + resource_reaches_status( + glance.images, + image.id, + expected_stat='active', + msg='Image status wait') + + return image + + +def create_image(glance, image_url, image_name, image_cache_dir='tests'): + """Download the latest cirros image and upload it to glance, + validate and return a resource pointer. + + :param glance: pointer to authenticated glance connection + :type glance: glanceclient.Client + :param image_url: URL to download image from + :type image_url: str + :param image_name: display name for new image + :type image_name: str + :param image_cache_dir: Directory to store image in before uploading + :type image_cache_dir: str + :returns: glance image pointer + :rtype: juju.unit.Unit or None + """ + logging.debug('Creating glance cirros image ' + '({})...'.format(image_name)) + + img_name = os.path.basename(urllib.parse.urlparse(image_url).path) + local_path = os.path.join(image_cache_dir, img_name) + + if not os.path.exists(local_path): + download_image(image_url, local_path) + + image = upload_image_to_glance(glance, local_path, image_name) + return image