From 502b96c772150ae03dd8df28cdc489d6ddb79af2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 10:13:12 +0000 Subject: [PATCH] Add helps for charm functional tests The idea of this change is to put together helpers for including functional tests for charms in a central location. The charm declares the tests and bundles to be run to complete a functional tests but the tests themselves live here in zaza. To use this code the charm should have the following: 1) zaza in the test-requirements.txt 2) tox.ini should include a target like: ``` [testenv:func35] basepython = python3 commands = functest-bundle-deploy ``` 3) Bundles which are to be used for the tests: ``` ls -1 tests/bundles/* tests/bundles/xenial.yaml ``` 4) A tests/tests.yaml file that describes the bundles to be run and the tests ``` charm_name: vault tests: - zaza.charms_tests.vault.VaultTest gate_bundles: - xenial dev_bundles: - bionic ``` Tests can be run without running a deployment using functest-run-tests with the list of test classes to be run: ``` functest-run-tests -t zaza.charms_tests.vault.VaultTest ``` Known Issues: - The deploy_bundle and add_model methods should be using libjuju - DEV_BUNDLES is currently ignored - VaultUtils and VaultTest should probably be in separate files. - When skipIfNotHA skips a test unittest does not pick up it has been skipped - A new model is created for each bundle even if an existing empty model exists - No model cleanup is performed. --- setup.py | 8 +- zaza/charms_tests/__init__.py | 0 zaza/charms_tests/test_utils.py | 15 +++ zaza/charms_tests/vault.py | 182 ++++++++++++++++++++++++++++++++ zaza/functests/__init__.py | 0 zaza/functests/deploy.py | 62 +++++++++++ zaza/model.py | 16 +++ 7 files changed, 281 insertions(+), 2 deletions(-) create mode 100644 zaza/charms_tests/__init__.py create mode 100644 zaza/charms_tests/test_utils.py create mode 100755 zaza/charms_tests/vault.py create mode 100644 zaza/functests/__init__.py create mode 100755 zaza/functests/deploy.py diff --git a/setup.py b/setup.py index a9374e1..a6ff7ec 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,9 @@ from setuptools.command.test import test as TestCommand version = "0.0.1.dev1" install_require = [ - 'juju' + 'juju', + 'juju-wait', + 'hvac' ] tests_require = [ @@ -72,6 +74,8 @@ setup( ], entry_points={ 'console_scripts': [ + 'functest-run-tests = zaza.functests.deploy:run_tests', + 'functest-bundle-deploy = zaza.functests.deploy:deploy', 'current-apps = zaza.model:main', 'tempest-config = zaza.tempest_config:main', ] @@ -88,4 +92,4 @@ setup( 'testing': tests_require, }, tests_require=tests_require, -) \ No newline at end of file +) diff --git a/zaza/charms_tests/__init__.py b/zaza/charms_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/charms_tests/test_utils.py b/zaza/charms_tests/test_utils.py new file mode 100644 index 0000000..02d5c6f --- /dev/null +++ b/zaza/charms_tests/test_utils.py @@ -0,0 +1,15 @@ +import logging +import unittest +import zaza.model + + +def skipIfNotHA(service_name): + def _skipIfNotHA_inner_1(f): + def _skipIfNotHA_inner_2(*args, **kwargs): + if len(zaza.model.unit_ips(service_name)) > 1: + return f(*args, **kwargs) + else: + logging.warn("Skipping HA test for non-ha service {}".format(service_name)) + return _skipIfNotHA_inner_2 + + return _skipIfNotHA_inner_1 diff --git a/zaza/charms_tests/vault.py b/zaza/charms_tests/vault.py new file mode 100755 index 0000000..175917b --- /dev/null +++ b/zaza/charms_tests/vault.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 + +import hvac +import logging +import os +import requests +import time +import unittest +import urllib3 +import uuid +import yaml + +import zaza.charms_tests.test_utils as test_utils +import zaza.model + + +class VaultUtils(object): + + def get_client(self, vault_url): + """Return an hvac client for the given URL + + :param vault_url: Vault url to point client at + :returns: hvac.Client + """ + return hvac.Client(url=vault_url) + + def init_vault(self, client, shares=1, threshold=1): + """Initialise vault + + :param client: hvac.Client Client to use for initiliasation + :param shares: int Number of key shares to create + :param threshold: int Number of keys needed to unseal vault + :returns: hvac.Client + """ + return client.initialize(shares, threshold) + + def get_clients(self, units=None): + """Create a list of clients, one per vault server + + :param units: [ip1, ip2, ...] List of IP addresses of vault endpoints + :returns: [hvac.Client, ...] List of clients + """ + if not units: + units = zaza.model.unit_ips('vault') + clients = [] + for unit in units: + vault_url = 'http://{}:8200'.format(unit) + clients.append((unit, self.get_client(vault_url))) + return clients + + def is_initialized(self, client): + """Check if vault is initialized + + :param client: hvac.Client Client to use to check if vault is + initialized + :returns: bool + """ + initialized = False + for i in range(1, 10): + try: + initialized = client[1].is_initialized() + except (ConnectionRefusedError, + urllib3.exceptions.NewConnectionError, + urllib3.exceptions.MaxRetryError, + requests.exceptions.ConnectionError): + time.sleep(2) + else: + break + else: + raise Exception("Cannot connect") + return initialized + + def get_credentails_from_file(self, auth_file): + """Read the vault credentials from the auth_file + + :param auth_file: str Path to file with credentials + :returns: {} Dictionary of credentials + """ + with open(auth_file, 'r') as stream: + vault_creds = yaml.load(stream) + return vault_creds + + def write_credentails(self, auth_file, vault_creds): + """Write the vault credentials to the auth_file + + :param auth_file: str Path to file to write credentials + """ + with open(auth_file, 'w') as outfile: + yaml.dump(vault_creds, outfile, default_flow_style=False) + + def unseal_all(self, clients, key): + """Unseal all the vaults with the given clients with the provided key + + :param clients: [hvac.Client, ...] List of clients + :param key: str key to unlock clients + """ + for (addr, client) in clients: + if client.is_sealed(): + client.unseal(key) + + def auth_all(self, clients, token): + """Authenticate all the given clients with the provided token + + :param clients: [hvac.Client, ...] List of clients + :param token: str token to authorize clients + """ + for (addr, client) in clients: + client.token = token + + +class VaultTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + vutils = VaultUtils() + cls.clients = vutils.get_clients() + unseal_client = cls.clients[0] + initialized = vutils.is_initialized(unseal_client) + # The credentials are written to a file to allow the tests to be re-run + # this is mainly useful for manually working on the tests. + auth_file = "/tmp/vault_tests.yaml" + if initialized: + vault_creds = vutils.get_credentails_from_file(auth_file) + else: + vault_creds = vutils.init_vault(unseal_client[1]) + vutils.write_credentails(auth_file, vault_creds) + vutils.unseal_all(cls.clients, vault_creds['keys'][0]) + vutils.auth_all(cls.clients, vault_creds['root_token']) + + def test_all_clients_authenticated(self): + for (addr, client) in self.clients: + for i in range(1, 10): + try: + self.assertTrue(client.is_authenticated()) + except hvac.exceptions.InternalServerError: + time.sleep(2) + else: + break + else: + self.assertTrue(client.is_authenticated()) + + def check_read(self, key, value): + for (addr, client) in self.clients: + self.assertEqual( + client.read('secret/uuids')['data']['uuid'], + value) + + def test_consistent_read_write(self): + key = 'secret/uuids' + for (addr, client) in self.clients: + value = str(uuid.uuid1()) + client.write(key, uuid=value, lease='1h') + # Now check all clients read the same value back + self.check_read(key, value) + + @test_utils.skipIfNotHA('vault') + def test_vault_ha_statuses(self): + leader = [] + leader_address = [] + leader_cluster_address = [] + for (addr, client) in self.clients: + self.assertTrue(client.ha_status['ha_enabled']) + leader_address.append( + client.ha_status['leader_address']) + leader_cluster_address.append( + client.ha_status['leader_cluster_address']) + if client.ha_status['is_self']: + leader.append(addr) + # Check there is exactly one leader + self.assertEqual(len(leader), 1) + # Check both cluster addresses match accross the cluster + self.assertEqual(len(set(leader_address)), 1) + self.assertEqual(len(set(leader_cluster_address)), 1) + + def test_check_vault_status(self): + for (addr, client) in self.clients: + self.assertFalse(client.seal_status['sealed']) + self.assertTrue(client.seal_status['cluster_name']) + + +if __name__ == '__main__': + unittest.main() diff --git a/zaza/functests/__init__.py b/zaza/functests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py new file mode 100755 index 0000000..a01605a --- /dev/null +++ b/zaza/functests/deploy.py @@ -0,0 +1,62 @@ +import argparse +import datetime +import importlib +import logging +import os +import subprocess +import sys +import unittest +import yaml + +import juju_wait + +BUNDLE_DIR = "./tests/bundles/" +DEFAULT_TEST_CONFIG = "./tests/tests.yaml" + +def deploy_bundle(bundle, model): + logging.info("Deploying bundle {}".format(bundle)) + subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) + +def add_model(model_name): + logging.info("Adding model {}".format(model_name)) + subprocess.check_call(['juju', 'add-model', model_name]) + +def get_test_class(class_str): + test_module_name = '.'.join(class_str.split('.')[:-1]) + test_class_name = class_str.split('.')[-1] + test_module = importlib.import_module(test_module_name) + return getattr(test_module, test_class_name) + +def run_test_list(tests): + for _testcase in tests: + testcase = get_test_class(_testcase) + suite = unittest.TestLoader().loadTestsFromTestCase(testcase) + test_result = unittest.TextTestRunner(verbosity=2).run(suite) + assert test_result.wasSuccessful(), "Test run failed" + +def deploy(): + test_config = get_test_config() + for t in test_config['gate_bundles']: + timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + model_name = '{}{}{}'.format(test_config['charm_name'], t, timestamp) + add_model(model_name) + deploy_bundle( + os.path.join(BUNDLE_DIR, '{}.yaml'.format(t)), + model_name) + logging.info("Waiting for environment to settle") + juju_wait.wait() + run_test_list(test_config['tests']) + +def get_test_config(): + with open(DEFAULT_TEST_CONFIG, 'r') as stream: + return yaml.load(stream) + +def run_tests(): + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-t','--tests', nargs='+', + help='Space sperated list of test classes', + required=False) + args = parser.parse_args() + tests = args.tests or get_test_config()['tests'] + run_test_list(tests) diff --git a/zaza/model.py b/zaza/model.py index db10db5..8f7c726 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -19,6 +19,22 @@ async def deployed(filter=None): await model.disconnect() +async def _unit_ips(service_name): + # Create a Model instance. We need to connect our Model to a Juju api + # server before we can use it. + model = Model() + # Connect to the currently active Juju model + await model.connect_current() + app = model.applications[service_name] + ips = [] + for unit in app.units: + ips.append(unit.public_address) + await model.disconnect() + return ips + +def unit_ips(service_name): + return loop.run(_unit_ips(service_name)) + def main(): # Run the deploy coroutine in an asyncio event loop, using a helper # that abstracts loop creation and teardown.