From 502b96c772150ae03dd8df28cdc489d6ddb79af2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 10:13:12 +0000 Subject: [PATCH 01/21] Add helps for charm functional tests The idea of this change is to put together helpers for including functional tests for charms in a central location. The charm declares the tests and bundles to be run to complete a functional tests but the tests themselves live here in zaza. To use this code the charm should have the following: 1) zaza in the test-requirements.txt 2) tox.ini should include a target like: ``` [testenv:func35] basepython = python3 commands = functest-bundle-deploy ``` 3) Bundles which are to be used for the tests: ``` ls -1 tests/bundles/* tests/bundles/xenial.yaml ``` 4) A tests/tests.yaml file that describes the bundles to be run and the tests ``` charm_name: vault tests: - zaza.charms_tests.vault.VaultTest gate_bundles: - xenial dev_bundles: - bionic ``` Tests can be run without running a deployment using functest-run-tests with the list of test classes to be run: ``` functest-run-tests -t zaza.charms_tests.vault.VaultTest ``` Known Issues: - The deploy_bundle and add_model methods should be using libjuju - DEV_BUNDLES is currently ignored - VaultUtils and VaultTest should probably be in separate files. - When skipIfNotHA skips a test unittest does not pick up it has been skipped - A new model is created for each bundle even if an existing empty model exists - No model cleanup is performed. --- setup.py | 8 +- zaza/charms_tests/__init__.py | 0 zaza/charms_tests/test_utils.py | 15 +++ zaza/charms_tests/vault.py | 182 ++++++++++++++++++++++++++++++++ zaza/functests/__init__.py | 0 zaza/functests/deploy.py | 62 +++++++++++ zaza/model.py | 16 +++ 7 files changed, 281 insertions(+), 2 deletions(-) create mode 100644 zaza/charms_tests/__init__.py create mode 100644 zaza/charms_tests/test_utils.py create mode 100755 zaza/charms_tests/vault.py create mode 100644 zaza/functests/__init__.py create mode 100755 zaza/functests/deploy.py diff --git a/setup.py b/setup.py index a9374e1..a6ff7ec 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,9 @@ from setuptools.command.test import test as TestCommand version = "0.0.1.dev1" install_require = [ - 'juju' + 'juju', + 'juju-wait', + 'hvac' ] tests_require = [ @@ -72,6 +74,8 @@ setup( ], entry_points={ 'console_scripts': [ + 'functest-run-tests = zaza.functests.deploy:run_tests', + 'functest-bundle-deploy = zaza.functests.deploy:deploy', 'current-apps = zaza.model:main', 'tempest-config = zaza.tempest_config:main', ] @@ -88,4 +92,4 @@ setup( 'testing': tests_require, }, tests_require=tests_require, -) \ No newline at end of file +) diff --git a/zaza/charms_tests/__init__.py b/zaza/charms_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/charms_tests/test_utils.py b/zaza/charms_tests/test_utils.py new file mode 100644 index 0000000..02d5c6f --- /dev/null +++ b/zaza/charms_tests/test_utils.py @@ -0,0 +1,15 @@ +import logging +import unittest +import zaza.model + + +def skipIfNotHA(service_name): + def _skipIfNotHA_inner_1(f): + def _skipIfNotHA_inner_2(*args, **kwargs): + if len(zaza.model.unit_ips(service_name)) > 1: + return f(*args, **kwargs) + else: + logging.warn("Skipping HA test for non-ha service {}".format(service_name)) + return _skipIfNotHA_inner_2 + + return _skipIfNotHA_inner_1 diff --git a/zaza/charms_tests/vault.py b/zaza/charms_tests/vault.py new file mode 100755 index 0000000..175917b --- /dev/null +++ b/zaza/charms_tests/vault.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 + +import hvac +import logging +import os +import requests +import time +import unittest +import urllib3 +import uuid +import yaml + +import zaza.charms_tests.test_utils as test_utils +import zaza.model + + +class VaultUtils(object): + + def get_client(self, vault_url): + """Return an hvac client for the given URL + + :param vault_url: Vault url to point client at + :returns: hvac.Client + """ + return hvac.Client(url=vault_url) + + def init_vault(self, client, shares=1, threshold=1): + """Initialise vault + + :param client: hvac.Client Client to use for initiliasation + :param shares: int Number of key shares to create + :param threshold: int Number of keys needed to unseal vault + :returns: hvac.Client + """ + return client.initialize(shares, threshold) + + def get_clients(self, units=None): + """Create a list of clients, one per vault server + + :param units: [ip1, ip2, ...] List of IP addresses of vault endpoints + :returns: [hvac.Client, ...] List of clients + """ + if not units: + units = zaza.model.unit_ips('vault') + clients = [] + for unit in units: + vault_url = 'http://{}:8200'.format(unit) + clients.append((unit, self.get_client(vault_url))) + return clients + + def is_initialized(self, client): + """Check if vault is initialized + + :param client: hvac.Client Client to use to check if vault is + initialized + :returns: bool + """ + initialized = False + for i in range(1, 10): + try: + initialized = client[1].is_initialized() + except (ConnectionRefusedError, + urllib3.exceptions.NewConnectionError, + urllib3.exceptions.MaxRetryError, + requests.exceptions.ConnectionError): + time.sleep(2) + else: + break + else: + raise Exception("Cannot connect") + return initialized + + def get_credentails_from_file(self, auth_file): + """Read the vault credentials from the auth_file + + :param auth_file: str Path to file with credentials + :returns: {} Dictionary of credentials + """ + with open(auth_file, 'r') as stream: + vault_creds = yaml.load(stream) + return vault_creds + + def write_credentails(self, auth_file, vault_creds): + """Write the vault credentials to the auth_file + + :param auth_file: str Path to file to write credentials + """ + with open(auth_file, 'w') as outfile: + yaml.dump(vault_creds, outfile, default_flow_style=False) + + def unseal_all(self, clients, key): + """Unseal all the vaults with the given clients with the provided key + + :param clients: [hvac.Client, ...] List of clients + :param key: str key to unlock clients + """ + for (addr, client) in clients: + if client.is_sealed(): + client.unseal(key) + + def auth_all(self, clients, token): + """Authenticate all the given clients with the provided token + + :param clients: [hvac.Client, ...] List of clients + :param token: str token to authorize clients + """ + for (addr, client) in clients: + client.token = token + + +class VaultTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + vutils = VaultUtils() + cls.clients = vutils.get_clients() + unseal_client = cls.clients[0] + initialized = vutils.is_initialized(unseal_client) + # The credentials are written to a file to allow the tests to be re-run + # this is mainly useful for manually working on the tests. + auth_file = "/tmp/vault_tests.yaml" + if initialized: + vault_creds = vutils.get_credentails_from_file(auth_file) + else: + vault_creds = vutils.init_vault(unseal_client[1]) + vutils.write_credentails(auth_file, vault_creds) + vutils.unseal_all(cls.clients, vault_creds['keys'][0]) + vutils.auth_all(cls.clients, vault_creds['root_token']) + + def test_all_clients_authenticated(self): + for (addr, client) in self.clients: + for i in range(1, 10): + try: + self.assertTrue(client.is_authenticated()) + except hvac.exceptions.InternalServerError: + time.sleep(2) + else: + break + else: + self.assertTrue(client.is_authenticated()) + + def check_read(self, key, value): + for (addr, client) in self.clients: + self.assertEqual( + client.read('secret/uuids')['data']['uuid'], + value) + + def test_consistent_read_write(self): + key = 'secret/uuids' + for (addr, client) in self.clients: + value = str(uuid.uuid1()) + client.write(key, uuid=value, lease='1h') + # Now check all clients read the same value back + self.check_read(key, value) + + @test_utils.skipIfNotHA('vault') + def test_vault_ha_statuses(self): + leader = [] + leader_address = [] + leader_cluster_address = [] + for (addr, client) in self.clients: + self.assertTrue(client.ha_status['ha_enabled']) + leader_address.append( + client.ha_status['leader_address']) + leader_cluster_address.append( + client.ha_status['leader_cluster_address']) + if client.ha_status['is_self']: + leader.append(addr) + # Check there is exactly one leader + self.assertEqual(len(leader), 1) + # Check both cluster addresses match accross the cluster + self.assertEqual(len(set(leader_address)), 1) + self.assertEqual(len(set(leader_cluster_address)), 1) + + def test_check_vault_status(self): + for (addr, client) in self.clients: + self.assertFalse(client.seal_status['sealed']) + self.assertTrue(client.seal_status['cluster_name']) + + +if __name__ == '__main__': + unittest.main() diff --git a/zaza/functests/__init__.py b/zaza/functests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py new file mode 100755 index 0000000..a01605a --- /dev/null +++ b/zaza/functests/deploy.py @@ -0,0 +1,62 @@ +import argparse +import datetime +import importlib +import logging +import os +import subprocess +import sys +import unittest +import yaml + +import juju_wait + +BUNDLE_DIR = "./tests/bundles/" +DEFAULT_TEST_CONFIG = "./tests/tests.yaml" + +def deploy_bundle(bundle, model): + logging.info("Deploying bundle {}".format(bundle)) + subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) + +def add_model(model_name): + logging.info("Adding model {}".format(model_name)) + subprocess.check_call(['juju', 'add-model', model_name]) + +def get_test_class(class_str): + test_module_name = '.'.join(class_str.split('.')[:-1]) + test_class_name = class_str.split('.')[-1] + test_module = importlib.import_module(test_module_name) + return getattr(test_module, test_class_name) + +def run_test_list(tests): + for _testcase in tests: + testcase = get_test_class(_testcase) + suite = unittest.TestLoader().loadTestsFromTestCase(testcase) + test_result = unittest.TextTestRunner(verbosity=2).run(suite) + assert test_result.wasSuccessful(), "Test run failed" + +def deploy(): + test_config = get_test_config() + for t in test_config['gate_bundles']: + timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + model_name = '{}{}{}'.format(test_config['charm_name'], t, timestamp) + add_model(model_name) + deploy_bundle( + os.path.join(BUNDLE_DIR, '{}.yaml'.format(t)), + model_name) + logging.info("Waiting for environment to settle") + juju_wait.wait() + run_test_list(test_config['tests']) + +def get_test_config(): + with open(DEFAULT_TEST_CONFIG, 'r') as stream: + return yaml.load(stream) + +def run_tests(): + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-t','--tests', nargs='+', + help='Space sperated list of test classes', + required=False) + args = parser.parse_args() + tests = args.tests or get_test_config()['tests'] + run_test_list(tests) diff --git a/zaza/model.py b/zaza/model.py index db10db5..8f7c726 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -19,6 +19,22 @@ async def deployed(filter=None): await model.disconnect() +async def _unit_ips(service_name): + # Create a Model instance. We need to connect our Model to a Juju api + # server before we can use it. + model = Model() + # Connect to the currently active Juju model + await model.connect_current() + app = model.applications[service_name] + ips = [] + for unit in app.units: + ips.append(unit.public_address) + await model.disconnect() + return ips + +def unit_ips(service_name): + return loop.run(_unit_ips(service_name)) + def main(): # Run the deploy coroutine in an asyncio event loop, using a helper # that abstracts loop creation and teardown. From b36c9bde254a1d2575aaad50f964cc7e13bfaf85 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 10:46:14 +0000 Subject: [PATCH 02/21] Removing vault specific tests to cleanup mp --- zaza/charms_tests/test_utils.py | 15 --- zaza/charms_tests/vault.py | 182 -------------------------------- 2 files changed, 197 deletions(-) delete mode 100644 zaza/charms_tests/test_utils.py delete mode 100755 zaza/charms_tests/vault.py diff --git a/zaza/charms_tests/test_utils.py b/zaza/charms_tests/test_utils.py deleted file mode 100644 index 02d5c6f..0000000 --- a/zaza/charms_tests/test_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging -import unittest -import zaza.model - - -def skipIfNotHA(service_name): - def _skipIfNotHA_inner_1(f): - def _skipIfNotHA_inner_2(*args, **kwargs): - if len(zaza.model.unit_ips(service_name)) > 1: - return f(*args, **kwargs) - else: - logging.warn("Skipping HA test for non-ha service {}".format(service_name)) - return _skipIfNotHA_inner_2 - - return _skipIfNotHA_inner_1 diff --git a/zaza/charms_tests/vault.py b/zaza/charms_tests/vault.py deleted file mode 100755 index 175917b..0000000 --- a/zaza/charms_tests/vault.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python3 - -import hvac -import logging -import os -import requests -import time -import unittest -import urllib3 -import uuid -import yaml - -import zaza.charms_tests.test_utils as test_utils -import zaza.model - - -class VaultUtils(object): - - def get_client(self, vault_url): - """Return an hvac client for the given URL - - :param vault_url: Vault url to point client at - :returns: hvac.Client - """ - return hvac.Client(url=vault_url) - - def init_vault(self, client, shares=1, threshold=1): - """Initialise vault - - :param client: hvac.Client Client to use for initiliasation - :param shares: int Number of key shares to create - :param threshold: int Number of keys needed to unseal vault - :returns: hvac.Client - """ - return client.initialize(shares, threshold) - - def get_clients(self, units=None): - """Create a list of clients, one per vault server - - :param units: [ip1, ip2, ...] List of IP addresses of vault endpoints - :returns: [hvac.Client, ...] List of clients - """ - if not units: - units = zaza.model.unit_ips('vault') - clients = [] - for unit in units: - vault_url = 'http://{}:8200'.format(unit) - clients.append((unit, self.get_client(vault_url))) - return clients - - def is_initialized(self, client): - """Check if vault is initialized - - :param client: hvac.Client Client to use to check if vault is - initialized - :returns: bool - """ - initialized = False - for i in range(1, 10): - try: - initialized = client[1].is_initialized() - except (ConnectionRefusedError, - urllib3.exceptions.NewConnectionError, - urllib3.exceptions.MaxRetryError, - requests.exceptions.ConnectionError): - time.sleep(2) - else: - break - else: - raise Exception("Cannot connect") - return initialized - - def get_credentails_from_file(self, auth_file): - """Read the vault credentials from the auth_file - - :param auth_file: str Path to file with credentials - :returns: {} Dictionary of credentials - """ - with open(auth_file, 'r') as stream: - vault_creds = yaml.load(stream) - return vault_creds - - def write_credentails(self, auth_file, vault_creds): - """Write the vault credentials to the auth_file - - :param auth_file: str Path to file to write credentials - """ - with open(auth_file, 'w') as outfile: - yaml.dump(vault_creds, outfile, default_flow_style=False) - - def unseal_all(self, clients, key): - """Unseal all the vaults with the given clients with the provided key - - :param clients: [hvac.Client, ...] List of clients - :param key: str key to unlock clients - """ - for (addr, client) in clients: - if client.is_sealed(): - client.unseal(key) - - def auth_all(self, clients, token): - """Authenticate all the given clients with the provided token - - :param clients: [hvac.Client, ...] List of clients - :param token: str token to authorize clients - """ - for (addr, client) in clients: - client.token = token - - -class VaultTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - vutils = VaultUtils() - cls.clients = vutils.get_clients() - unseal_client = cls.clients[0] - initialized = vutils.is_initialized(unseal_client) - # The credentials are written to a file to allow the tests to be re-run - # this is mainly useful for manually working on the tests. - auth_file = "/tmp/vault_tests.yaml" - if initialized: - vault_creds = vutils.get_credentails_from_file(auth_file) - else: - vault_creds = vutils.init_vault(unseal_client[1]) - vutils.write_credentails(auth_file, vault_creds) - vutils.unseal_all(cls.clients, vault_creds['keys'][0]) - vutils.auth_all(cls.clients, vault_creds['root_token']) - - def test_all_clients_authenticated(self): - for (addr, client) in self.clients: - for i in range(1, 10): - try: - self.assertTrue(client.is_authenticated()) - except hvac.exceptions.InternalServerError: - time.sleep(2) - else: - break - else: - self.assertTrue(client.is_authenticated()) - - def check_read(self, key, value): - for (addr, client) in self.clients: - self.assertEqual( - client.read('secret/uuids')['data']['uuid'], - value) - - def test_consistent_read_write(self): - key = 'secret/uuids' - for (addr, client) in self.clients: - value = str(uuid.uuid1()) - client.write(key, uuid=value, lease='1h') - # Now check all clients read the same value back - self.check_read(key, value) - - @test_utils.skipIfNotHA('vault') - def test_vault_ha_statuses(self): - leader = [] - leader_address = [] - leader_cluster_address = [] - for (addr, client) in self.clients: - self.assertTrue(client.ha_status['ha_enabled']) - leader_address.append( - client.ha_status['leader_address']) - leader_cluster_address.append( - client.ha_status['leader_cluster_address']) - if client.ha_status['is_self']: - leader.append(addr) - # Check there is exactly one leader - self.assertEqual(len(leader), 1) - # Check both cluster addresses match accross the cluster - self.assertEqual(len(set(leader_address)), 1) - self.assertEqual(len(set(leader_cluster_address)), 1) - - def test_check_vault_status(self): - for (addr, client) in self.clients: - self.assertFalse(client.seal_status['sealed']) - self.assertTrue(client.seal_status['cluster_name']) - - -if __name__ == '__main__': - unittest.main() From 5e613051bcdff28389aadde952bd926588de4f4c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 10:48:29 +0000 Subject: [PATCH 03/21] Remove new unit_ips from mp, will add in as a seperate mp --- zaza/model.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/zaza/model.py b/zaza/model.py index 8f7c726..db10db5 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -19,22 +19,6 @@ async def deployed(filter=None): await model.disconnect() -async def _unit_ips(service_name): - # Create a Model instance. We need to connect our Model to a Juju api - # server before we can use it. - model = Model() - # Connect to the currently active Juju model - await model.connect_current() - app = model.applications[service_name] - ips = [] - for unit in app.units: - ips.append(unit.public_address) - await model.disconnect() - return ips - -def unit_ips(service_name): - return loop.run(_unit_ips(service_name)) - def main(): # Run the deploy coroutine in an asyncio event loop, using a helper # that abstracts loop creation and teardown. From c7e30d2516b7d00597071877c76d0f4920bb8cb8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 11:30:44 +0000 Subject: [PATCH 04/21] Add docstrings --- zaza/functests/deploy.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py index a01605a..56b4c00 100755 --- a/zaza/functests/deploy.py +++ b/zaza/functests/deploy.py @@ -14,20 +14,37 @@ BUNDLE_DIR = "./tests/bundles/" DEFAULT_TEST_CONFIG = "./tests/tests.yaml" def deploy_bundle(bundle, model): + """Deploy the given bundle file in the specified model + + :param bundle: str Path to bundle file + :param model: str Name of model to deploy bundle in + """ logging.info("Deploying bundle {}".format(bundle)) subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) def add_model(model_name): + """Add a model with the given name + + :param model: str Name of model to add + """ logging.info("Adding model {}".format(model_name)) subprocess.check_call(['juju', 'add-model', model_name]) def get_test_class(class_str): + """Get the test class represented by the given string + + :param class_str: str Class to be returned + """ test_module_name = '.'.join(class_str.split('.')[:-1]) test_class_name = class_str.split('.')[-1] test_module = importlib.import_module(test_module_name) return getattr(test_module, test_class_name) def run_test_list(tests): + """Run the tests as defined in the list of test classes in series. + + :param tests: [TestClass1, TestClass2, ...] List of tests classes to run + """ for _testcase in tests: testcase = get_test_class(_testcase) suite = unittest.TestLoader().loadTestsFromTestCase(testcase) @@ -35,6 +52,9 @@ def run_test_list(tests): assert test_result.wasSuccessful(), "Test run failed" def deploy(): + """Deploy the bundles and run the tests as defined by the charms tests.yaml + + """ test_config = get_test_config() for t in test_config['gate_bundles']: timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') @@ -48,10 +68,15 @@ def deploy(): run_test_list(test_config['tests']) def get_test_config(): + """Read the yaml test config file and return the resulting config + + """ with open(DEFAULT_TEST_CONFIG, 'r') as stream: return yaml.load(stream) def run_tests(): + """Run the tests defined by the command line args or if none were provided + read the tests from the charms tests.yaml config file""" logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser() parser.add_argument('-t','--tests', nargs='+', From a7bba97b95bbdad852bb1a523a5b529a4df6c0dc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 11:38:41 +0000 Subject: [PATCH 05/21] More docstraing fixes --- zaza/functests/deploy.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py index 56b4c00..8078a39 100755 --- a/zaza/functests/deploy.py +++ b/zaza/functests/deploy.py @@ -33,7 +33,9 @@ def add_model(model_name): def get_test_class(class_str): """Get the test class represented by the given string - :param class_str: str Class to be returned + :param class_str: str Class to be returned eg + get_test_class('zaza.charms_tests.svc.TestSVCClass1') returns + zaza.charms_tests.svc.TestSVCClass1 """ test_module_name = '.'.join(class_str.split('.')[:-1]) test_class_name = class_str.split('.')[-1] @@ -43,7 +45,8 @@ def get_test_class(class_str): def run_test_list(tests): """Run the tests as defined in the list of test classes in series. - :param tests: [TestClass1, TestClass2, ...] List of tests classes to run + :param tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] List of test + class strings """ for _testcase in tests: testcase = get_test_class(_testcase) From 7b7f616fe97f89bfdf203277d031b52d8b3695b0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 14:16:38 +0000 Subject: [PATCH 06/21] Add type info to doc strings --- zaza/functests/deploy.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py index 8078a39..0cd09d9 100755 --- a/zaza/functests/deploy.py +++ b/zaza/functests/deploy.py @@ -16,8 +16,10 @@ DEFAULT_TEST_CONFIG = "./tests/tests.yaml" def deploy_bundle(bundle, model): """Deploy the given bundle file in the specified model - :param bundle: str Path to bundle file - :param model: str Name of model to deploy bundle in + :param bundle: Path to bundle file + :type bundle: str + :param model: Name of model to deploy bundle in + :type model: str """ logging.info("Deploying bundle {}".format(bundle)) subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) @@ -25,7 +27,8 @@ def deploy_bundle(bundle, model): def add_model(model_name): """Add a model with the given name - :param model: str Name of model to add + :param model: Name of model to add + :type bundle: str """ logging.info("Adding model {}".format(model_name)) subprocess.check_call(['juju', 'add-model', model_name]) @@ -33,9 +36,13 @@ def add_model(model_name): def get_test_class(class_str): """Get the test class represented by the given string - :param class_str: str Class to be returned eg - get_test_class('zaza.charms_tests.svc.TestSVCClass1') returns - zaza.charms_tests.svc.TestSVCClass1 + For example, get_test_class('zaza.charms_tests.svc.TestSVCClass1') + returns zaza.charms_tests.svc.TestSVCClass1 + + :param class_str: Class to be returned + :type class_str: str + :returns: Test class + :rtype: class """ test_module_name = '.'.join(class_str.split('.')[:-1]) test_class_name = class_str.split('.')[-1] @@ -45,8 +52,9 @@ def get_test_class(class_str): def run_test_list(tests): """Run the tests as defined in the list of test classes in series. - :param tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] List of test - class strings + :param tests: List of test class strings + :type tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] + :raises: AssertionError if test run fails """ for _testcase in tests: testcase = get_test_class(_testcase) @@ -56,7 +64,6 @@ def run_test_list(tests): def deploy(): """Deploy the bundles and run the tests as defined by the charms tests.yaml - """ test_config = get_test_config() for t in test_config['gate_bundles']: @@ -73,6 +80,8 @@ def deploy(): def get_test_config(): """Read the yaml test config file and return the resulting config + :returns: Config dictionary + :rtype: dict """ with open(DEFAULT_TEST_CONFIG, 'r') as stream: return yaml.load(stream) From 458a76f05afe4dee269514cfd012956a576874b2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 10:13:12 +0000 Subject: [PATCH 07/21] Add helps for charm functional tests The idea of this change is to put together helpers for including functional tests for charms in a central location. The charm declares the tests and bundles to be run to complete a functional tests but the tests themselves live here in zaza. To use this code the charm should have the following: 1) zaza in the test-requirements.txt 2) tox.ini should include a target like: ``` [testenv:func35] basepython = python3 commands = functest-bundle-deploy ``` 3) Bundles which are to be used for the tests: ``` ls -1 tests/bundles/* tests/bundles/xenial.yaml ``` 4) A tests/tests.yaml file that describes the bundles to be run and the tests ``` charm_name: vault tests: - zaza.charms_tests.vault.VaultTest gate_bundles: - xenial dev_bundles: - bionic ``` Tests can be run without running a deployment using functest-run-tests with the list of test classes to be run: ``` functest-run-tests -t zaza.charms_tests.vault.VaultTest ``` Known Issues: - The deploy_bundle and add_model methods should be using libjuju - DEV_BUNDLES is currently ignored - VaultUtils and VaultTest should probably be in separate files. - When skipIfNotHA skips a test unittest does not pick up it has been skipped - A new model is created for each bundle even if an existing empty model exists - No model cleanup is performed. --- zaza/charms_tests/test_utils.py | 15 +++ zaza/charms_tests/vault.py | 182 ++++++++++++++++++++++++++++++++ zaza/functests/deploy.py | 99 ----------------- zaza/model.py | 16 +++ 4 files changed, 213 insertions(+), 99 deletions(-) create mode 100644 zaza/charms_tests/test_utils.py create mode 100755 zaza/charms_tests/vault.py delete mode 100755 zaza/functests/deploy.py diff --git a/zaza/charms_tests/test_utils.py b/zaza/charms_tests/test_utils.py new file mode 100644 index 0000000..02d5c6f --- /dev/null +++ b/zaza/charms_tests/test_utils.py @@ -0,0 +1,15 @@ +import logging +import unittest +import zaza.model + + +def skipIfNotHA(service_name): + def _skipIfNotHA_inner_1(f): + def _skipIfNotHA_inner_2(*args, **kwargs): + if len(zaza.model.unit_ips(service_name)) > 1: + return f(*args, **kwargs) + else: + logging.warn("Skipping HA test for non-ha service {}".format(service_name)) + return _skipIfNotHA_inner_2 + + return _skipIfNotHA_inner_1 diff --git a/zaza/charms_tests/vault.py b/zaza/charms_tests/vault.py new file mode 100755 index 0000000..175917b --- /dev/null +++ b/zaza/charms_tests/vault.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 + +import hvac +import logging +import os +import requests +import time +import unittest +import urllib3 +import uuid +import yaml + +import zaza.charms_tests.test_utils as test_utils +import zaza.model + + +class VaultUtils(object): + + def get_client(self, vault_url): + """Return an hvac client for the given URL + + :param vault_url: Vault url to point client at + :returns: hvac.Client + """ + return hvac.Client(url=vault_url) + + def init_vault(self, client, shares=1, threshold=1): + """Initialise vault + + :param client: hvac.Client Client to use for initiliasation + :param shares: int Number of key shares to create + :param threshold: int Number of keys needed to unseal vault + :returns: hvac.Client + """ + return client.initialize(shares, threshold) + + def get_clients(self, units=None): + """Create a list of clients, one per vault server + + :param units: [ip1, ip2, ...] List of IP addresses of vault endpoints + :returns: [hvac.Client, ...] List of clients + """ + if not units: + units = zaza.model.unit_ips('vault') + clients = [] + for unit in units: + vault_url = 'http://{}:8200'.format(unit) + clients.append((unit, self.get_client(vault_url))) + return clients + + def is_initialized(self, client): + """Check if vault is initialized + + :param client: hvac.Client Client to use to check if vault is + initialized + :returns: bool + """ + initialized = False + for i in range(1, 10): + try: + initialized = client[1].is_initialized() + except (ConnectionRefusedError, + urllib3.exceptions.NewConnectionError, + urllib3.exceptions.MaxRetryError, + requests.exceptions.ConnectionError): + time.sleep(2) + else: + break + else: + raise Exception("Cannot connect") + return initialized + + def get_credentails_from_file(self, auth_file): + """Read the vault credentials from the auth_file + + :param auth_file: str Path to file with credentials + :returns: {} Dictionary of credentials + """ + with open(auth_file, 'r') as stream: + vault_creds = yaml.load(stream) + return vault_creds + + def write_credentails(self, auth_file, vault_creds): + """Write the vault credentials to the auth_file + + :param auth_file: str Path to file to write credentials + """ + with open(auth_file, 'w') as outfile: + yaml.dump(vault_creds, outfile, default_flow_style=False) + + def unseal_all(self, clients, key): + """Unseal all the vaults with the given clients with the provided key + + :param clients: [hvac.Client, ...] List of clients + :param key: str key to unlock clients + """ + for (addr, client) in clients: + if client.is_sealed(): + client.unseal(key) + + def auth_all(self, clients, token): + """Authenticate all the given clients with the provided token + + :param clients: [hvac.Client, ...] List of clients + :param token: str token to authorize clients + """ + for (addr, client) in clients: + client.token = token + + +class VaultTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + vutils = VaultUtils() + cls.clients = vutils.get_clients() + unseal_client = cls.clients[0] + initialized = vutils.is_initialized(unseal_client) + # The credentials are written to a file to allow the tests to be re-run + # this is mainly useful for manually working on the tests. + auth_file = "/tmp/vault_tests.yaml" + if initialized: + vault_creds = vutils.get_credentails_from_file(auth_file) + else: + vault_creds = vutils.init_vault(unseal_client[1]) + vutils.write_credentails(auth_file, vault_creds) + vutils.unseal_all(cls.clients, vault_creds['keys'][0]) + vutils.auth_all(cls.clients, vault_creds['root_token']) + + def test_all_clients_authenticated(self): + for (addr, client) in self.clients: + for i in range(1, 10): + try: + self.assertTrue(client.is_authenticated()) + except hvac.exceptions.InternalServerError: + time.sleep(2) + else: + break + else: + self.assertTrue(client.is_authenticated()) + + def check_read(self, key, value): + for (addr, client) in self.clients: + self.assertEqual( + client.read('secret/uuids')['data']['uuid'], + value) + + def test_consistent_read_write(self): + key = 'secret/uuids' + for (addr, client) in self.clients: + value = str(uuid.uuid1()) + client.write(key, uuid=value, lease='1h') + # Now check all clients read the same value back + self.check_read(key, value) + + @test_utils.skipIfNotHA('vault') + def test_vault_ha_statuses(self): + leader = [] + leader_address = [] + leader_cluster_address = [] + for (addr, client) in self.clients: + self.assertTrue(client.ha_status['ha_enabled']) + leader_address.append( + client.ha_status['leader_address']) + leader_cluster_address.append( + client.ha_status['leader_cluster_address']) + if client.ha_status['is_self']: + leader.append(addr) + # Check there is exactly one leader + self.assertEqual(len(leader), 1) + # Check both cluster addresses match accross the cluster + self.assertEqual(len(set(leader_address)), 1) + self.assertEqual(len(set(leader_cluster_address)), 1) + + def test_check_vault_status(self): + for (addr, client) in self.clients: + self.assertFalse(client.seal_status['sealed']) + self.assertTrue(client.seal_status['cluster_name']) + + +if __name__ == '__main__': + unittest.main() diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py deleted file mode 100755 index 0cd09d9..0000000 --- a/zaza/functests/deploy.py +++ /dev/null @@ -1,99 +0,0 @@ -import argparse -import datetime -import importlib -import logging -import os -import subprocess -import sys -import unittest -import yaml - -import juju_wait - -BUNDLE_DIR = "./tests/bundles/" -DEFAULT_TEST_CONFIG = "./tests/tests.yaml" - -def deploy_bundle(bundle, model): - """Deploy the given bundle file in the specified model - - :param bundle: Path to bundle file - :type bundle: str - :param model: Name of model to deploy bundle in - :type model: str - """ - logging.info("Deploying bundle {}".format(bundle)) - subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) - -def add_model(model_name): - """Add a model with the given name - - :param model: Name of model to add - :type bundle: str - """ - logging.info("Adding model {}".format(model_name)) - subprocess.check_call(['juju', 'add-model', model_name]) - -def get_test_class(class_str): - """Get the test class represented by the given string - - For example, get_test_class('zaza.charms_tests.svc.TestSVCClass1') - returns zaza.charms_tests.svc.TestSVCClass1 - - :param class_str: Class to be returned - :type class_str: str - :returns: Test class - :rtype: class - """ - test_module_name = '.'.join(class_str.split('.')[:-1]) - test_class_name = class_str.split('.')[-1] - test_module = importlib.import_module(test_module_name) - return getattr(test_module, test_class_name) - -def run_test_list(tests): - """Run the tests as defined in the list of test classes in series. - - :param tests: List of test class strings - :type tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] - :raises: AssertionError if test run fails - """ - for _testcase in tests: - testcase = get_test_class(_testcase) - suite = unittest.TestLoader().loadTestsFromTestCase(testcase) - test_result = unittest.TextTestRunner(verbosity=2).run(suite) - assert test_result.wasSuccessful(), "Test run failed" - -def deploy(): - """Deploy the bundles and run the tests as defined by the charms tests.yaml - """ - test_config = get_test_config() - for t in test_config['gate_bundles']: - timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') - model_name = '{}{}{}'.format(test_config['charm_name'], t, timestamp) - add_model(model_name) - deploy_bundle( - os.path.join(BUNDLE_DIR, '{}.yaml'.format(t)), - model_name) - logging.info("Waiting for environment to settle") - juju_wait.wait() - run_test_list(test_config['tests']) - -def get_test_config(): - """Read the yaml test config file and return the resulting config - - :returns: Config dictionary - :rtype: dict - """ - with open(DEFAULT_TEST_CONFIG, 'r') as stream: - return yaml.load(stream) - -def run_tests(): - """Run the tests defined by the command line args or if none were provided - read the tests from the charms tests.yaml config file""" - logging.basicConfig(level=logging.INFO) - parser = argparse.ArgumentParser() - parser.add_argument('-t','--tests', nargs='+', - help='Space sperated list of test classes', - required=False) - args = parser.parse_args() - tests = args.tests or get_test_config()['tests'] - run_test_list(tests) diff --git a/zaza/model.py b/zaza/model.py index db10db5..8f7c726 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -19,6 +19,22 @@ async def deployed(filter=None): await model.disconnect() +async def _unit_ips(service_name): + # Create a Model instance. We need to connect our Model to a Juju api + # server before we can use it. + model = Model() + # Connect to the currently active Juju model + await model.connect_current() + app = model.applications[service_name] + ips = [] + for unit in app.units: + ips.append(unit.public_address) + await model.disconnect() + return ips + +def unit_ips(service_name): + return loop.run(_unit_ips(service_name)) + def main(): # Run the deploy coroutine in an asyncio event loop, using a helper # that abstracts loop creation and teardown. From a69f65f3fc839483088394e3071527ad84287a85 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 10:46:14 +0000 Subject: [PATCH 08/21] Removing vault specific tests to cleanup mp --- zaza/charms_tests/test_utils.py | 15 --- zaza/charms_tests/vault.py | 182 -------------------------------- 2 files changed, 197 deletions(-) delete mode 100644 zaza/charms_tests/test_utils.py delete mode 100755 zaza/charms_tests/vault.py diff --git a/zaza/charms_tests/test_utils.py b/zaza/charms_tests/test_utils.py deleted file mode 100644 index 02d5c6f..0000000 --- a/zaza/charms_tests/test_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging -import unittest -import zaza.model - - -def skipIfNotHA(service_name): - def _skipIfNotHA_inner_1(f): - def _skipIfNotHA_inner_2(*args, **kwargs): - if len(zaza.model.unit_ips(service_name)) > 1: - return f(*args, **kwargs) - else: - logging.warn("Skipping HA test for non-ha service {}".format(service_name)) - return _skipIfNotHA_inner_2 - - return _skipIfNotHA_inner_1 diff --git a/zaza/charms_tests/vault.py b/zaza/charms_tests/vault.py deleted file mode 100755 index 175917b..0000000 --- a/zaza/charms_tests/vault.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python3 - -import hvac -import logging -import os -import requests -import time -import unittest -import urllib3 -import uuid -import yaml - -import zaza.charms_tests.test_utils as test_utils -import zaza.model - - -class VaultUtils(object): - - def get_client(self, vault_url): - """Return an hvac client for the given URL - - :param vault_url: Vault url to point client at - :returns: hvac.Client - """ - return hvac.Client(url=vault_url) - - def init_vault(self, client, shares=1, threshold=1): - """Initialise vault - - :param client: hvac.Client Client to use for initiliasation - :param shares: int Number of key shares to create - :param threshold: int Number of keys needed to unseal vault - :returns: hvac.Client - """ - return client.initialize(shares, threshold) - - def get_clients(self, units=None): - """Create a list of clients, one per vault server - - :param units: [ip1, ip2, ...] List of IP addresses of vault endpoints - :returns: [hvac.Client, ...] List of clients - """ - if not units: - units = zaza.model.unit_ips('vault') - clients = [] - for unit in units: - vault_url = 'http://{}:8200'.format(unit) - clients.append((unit, self.get_client(vault_url))) - return clients - - def is_initialized(self, client): - """Check if vault is initialized - - :param client: hvac.Client Client to use to check if vault is - initialized - :returns: bool - """ - initialized = False - for i in range(1, 10): - try: - initialized = client[1].is_initialized() - except (ConnectionRefusedError, - urllib3.exceptions.NewConnectionError, - urllib3.exceptions.MaxRetryError, - requests.exceptions.ConnectionError): - time.sleep(2) - else: - break - else: - raise Exception("Cannot connect") - return initialized - - def get_credentails_from_file(self, auth_file): - """Read the vault credentials from the auth_file - - :param auth_file: str Path to file with credentials - :returns: {} Dictionary of credentials - """ - with open(auth_file, 'r') as stream: - vault_creds = yaml.load(stream) - return vault_creds - - def write_credentails(self, auth_file, vault_creds): - """Write the vault credentials to the auth_file - - :param auth_file: str Path to file to write credentials - """ - with open(auth_file, 'w') as outfile: - yaml.dump(vault_creds, outfile, default_flow_style=False) - - def unseal_all(self, clients, key): - """Unseal all the vaults with the given clients with the provided key - - :param clients: [hvac.Client, ...] List of clients - :param key: str key to unlock clients - """ - for (addr, client) in clients: - if client.is_sealed(): - client.unseal(key) - - def auth_all(self, clients, token): - """Authenticate all the given clients with the provided token - - :param clients: [hvac.Client, ...] List of clients - :param token: str token to authorize clients - """ - for (addr, client) in clients: - client.token = token - - -class VaultTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - vutils = VaultUtils() - cls.clients = vutils.get_clients() - unseal_client = cls.clients[0] - initialized = vutils.is_initialized(unseal_client) - # The credentials are written to a file to allow the tests to be re-run - # this is mainly useful for manually working on the tests. - auth_file = "/tmp/vault_tests.yaml" - if initialized: - vault_creds = vutils.get_credentails_from_file(auth_file) - else: - vault_creds = vutils.init_vault(unseal_client[1]) - vutils.write_credentails(auth_file, vault_creds) - vutils.unseal_all(cls.clients, vault_creds['keys'][0]) - vutils.auth_all(cls.clients, vault_creds['root_token']) - - def test_all_clients_authenticated(self): - for (addr, client) in self.clients: - for i in range(1, 10): - try: - self.assertTrue(client.is_authenticated()) - except hvac.exceptions.InternalServerError: - time.sleep(2) - else: - break - else: - self.assertTrue(client.is_authenticated()) - - def check_read(self, key, value): - for (addr, client) in self.clients: - self.assertEqual( - client.read('secret/uuids')['data']['uuid'], - value) - - def test_consistent_read_write(self): - key = 'secret/uuids' - for (addr, client) in self.clients: - value = str(uuid.uuid1()) - client.write(key, uuid=value, lease='1h') - # Now check all clients read the same value back - self.check_read(key, value) - - @test_utils.skipIfNotHA('vault') - def test_vault_ha_statuses(self): - leader = [] - leader_address = [] - leader_cluster_address = [] - for (addr, client) in self.clients: - self.assertTrue(client.ha_status['ha_enabled']) - leader_address.append( - client.ha_status['leader_address']) - leader_cluster_address.append( - client.ha_status['leader_cluster_address']) - if client.ha_status['is_self']: - leader.append(addr) - # Check there is exactly one leader - self.assertEqual(len(leader), 1) - # Check both cluster addresses match accross the cluster - self.assertEqual(len(set(leader_address)), 1) - self.assertEqual(len(set(leader_cluster_address)), 1) - - def test_check_vault_status(self): - for (addr, client) in self.clients: - self.assertFalse(client.seal_status['sealed']) - self.assertTrue(client.seal_status['cluster_name']) - - -if __name__ == '__main__': - unittest.main() From 12c7363b34e9c735b4c9257bc79cef92311998a2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 10:48:29 +0000 Subject: [PATCH 09/21] Remove new unit_ips from mp, will add in as a seperate mp --- zaza/model.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/zaza/model.py b/zaza/model.py index 8f7c726..db10db5 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -19,22 +19,6 @@ async def deployed(filter=None): await model.disconnect() -async def _unit_ips(service_name): - # Create a Model instance. We need to connect our Model to a Juju api - # server before we can use it. - model = Model() - # Connect to the currently active Juju model - await model.connect_current() - app = model.applications[service_name] - ips = [] - for unit in app.units: - ips.append(unit.public_address) - await model.disconnect() - return ips - -def unit_ips(service_name): - return loop.run(_unit_ips(service_name)) - def main(): # Run the deploy coroutine in an asyncio event loop, using a helper # that abstracts loop creation and teardown. From cde3238afab7cbfe6c218cf9c0834474b9baaa5f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 11:30:44 +0000 Subject: [PATCH 10/21] Add docstrings --- zaza/functests/deploy.py | 87 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100755 zaza/functests/deploy.py diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py new file mode 100755 index 0000000..56b4c00 --- /dev/null +++ b/zaza/functests/deploy.py @@ -0,0 +1,87 @@ +import argparse +import datetime +import importlib +import logging +import os +import subprocess +import sys +import unittest +import yaml + +import juju_wait + +BUNDLE_DIR = "./tests/bundles/" +DEFAULT_TEST_CONFIG = "./tests/tests.yaml" + +def deploy_bundle(bundle, model): + """Deploy the given bundle file in the specified model + + :param bundle: str Path to bundle file + :param model: str Name of model to deploy bundle in + """ + logging.info("Deploying bundle {}".format(bundle)) + subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) + +def add_model(model_name): + """Add a model with the given name + + :param model: str Name of model to add + """ + logging.info("Adding model {}".format(model_name)) + subprocess.check_call(['juju', 'add-model', model_name]) + +def get_test_class(class_str): + """Get the test class represented by the given string + + :param class_str: str Class to be returned + """ + test_module_name = '.'.join(class_str.split('.')[:-1]) + test_class_name = class_str.split('.')[-1] + test_module = importlib.import_module(test_module_name) + return getattr(test_module, test_class_name) + +def run_test_list(tests): + """Run the tests as defined in the list of test classes in series. + + :param tests: [TestClass1, TestClass2, ...] List of tests classes to run + """ + for _testcase in tests: + testcase = get_test_class(_testcase) + suite = unittest.TestLoader().loadTestsFromTestCase(testcase) + test_result = unittest.TextTestRunner(verbosity=2).run(suite) + assert test_result.wasSuccessful(), "Test run failed" + +def deploy(): + """Deploy the bundles and run the tests as defined by the charms tests.yaml + + """ + test_config = get_test_config() + for t in test_config['gate_bundles']: + timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + model_name = '{}{}{}'.format(test_config['charm_name'], t, timestamp) + add_model(model_name) + deploy_bundle( + os.path.join(BUNDLE_DIR, '{}.yaml'.format(t)), + model_name) + logging.info("Waiting for environment to settle") + juju_wait.wait() + run_test_list(test_config['tests']) + +def get_test_config(): + """Read the yaml test config file and return the resulting config + + """ + with open(DEFAULT_TEST_CONFIG, 'r') as stream: + return yaml.load(stream) + +def run_tests(): + """Run the tests defined by the command line args or if none were provided + read the tests from the charms tests.yaml config file""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-t','--tests', nargs='+', + help='Space sperated list of test classes', + required=False) + args = parser.parse_args() + tests = args.tests or get_test_config()['tests'] + run_test_list(tests) From d61115a1e53be4f3114b0da7798c8e54b57302ff Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 11:38:41 +0000 Subject: [PATCH 11/21] More docstraing fixes --- zaza/functests/deploy.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py index 56b4c00..8078a39 100755 --- a/zaza/functests/deploy.py +++ b/zaza/functests/deploy.py @@ -33,7 +33,9 @@ def add_model(model_name): def get_test_class(class_str): """Get the test class represented by the given string - :param class_str: str Class to be returned + :param class_str: str Class to be returned eg + get_test_class('zaza.charms_tests.svc.TestSVCClass1') returns + zaza.charms_tests.svc.TestSVCClass1 """ test_module_name = '.'.join(class_str.split('.')[:-1]) test_class_name = class_str.split('.')[-1] @@ -43,7 +45,8 @@ def get_test_class(class_str): def run_test_list(tests): """Run the tests as defined in the list of test classes in series. - :param tests: [TestClass1, TestClass2, ...] List of tests classes to run + :param tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] List of test + class strings """ for _testcase in tests: testcase = get_test_class(_testcase) From 38c3710a8837c5e231854e000e3e84d9e77d2b34 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Mar 2018 14:16:38 +0000 Subject: [PATCH 12/21] Add type info to doc strings --- zaza/functests/deploy.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py index 8078a39..0cd09d9 100755 --- a/zaza/functests/deploy.py +++ b/zaza/functests/deploy.py @@ -16,8 +16,10 @@ DEFAULT_TEST_CONFIG = "./tests/tests.yaml" def deploy_bundle(bundle, model): """Deploy the given bundle file in the specified model - :param bundle: str Path to bundle file - :param model: str Name of model to deploy bundle in + :param bundle: Path to bundle file + :type bundle: str + :param model: Name of model to deploy bundle in + :type model: str """ logging.info("Deploying bundle {}".format(bundle)) subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) @@ -25,7 +27,8 @@ def deploy_bundle(bundle, model): def add_model(model_name): """Add a model with the given name - :param model: str Name of model to add + :param model: Name of model to add + :type bundle: str """ logging.info("Adding model {}".format(model_name)) subprocess.check_call(['juju', 'add-model', model_name]) @@ -33,9 +36,13 @@ def add_model(model_name): def get_test_class(class_str): """Get the test class represented by the given string - :param class_str: str Class to be returned eg - get_test_class('zaza.charms_tests.svc.TestSVCClass1') returns - zaza.charms_tests.svc.TestSVCClass1 + For example, get_test_class('zaza.charms_tests.svc.TestSVCClass1') + returns zaza.charms_tests.svc.TestSVCClass1 + + :param class_str: Class to be returned + :type class_str: str + :returns: Test class + :rtype: class """ test_module_name = '.'.join(class_str.split('.')[:-1]) test_class_name = class_str.split('.')[-1] @@ -45,8 +52,9 @@ def get_test_class(class_str): def run_test_list(tests): """Run the tests as defined in the list of test classes in series. - :param tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] List of test - class strings + :param tests: List of test class strings + :type tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] + :raises: AssertionError if test run fails """ for _testcase in tests: testcase = get_test_class(_testcase) @@ -56,7 +64,6 @@ def run_test_list(tests): def deploy(): """Deploy the bundles and run the tests as defined by the charms tests.yaml - """ test_config = get_test_config() for t in test_config['gate_bundles']: @@ -73,6 +80,8 @@ def deploy(): def get_test_config(): """Read the yaml test config file and return the resulting config + :returns: Config dictionary + :rtype: dict """ with open(DEFAULT_TEST_CONFIG, 'r') as stream: return yaml.load(stream) From 86b99c1134d173c9a5bfef7930761dbe1606a25a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 23 Mar 2018 17:19:44 +0000 Subject: [PATCH 13/21] Refactor code to seperate phases out --- setup.py | 8 +- zaza/charm_testing/README.md | 201 +++++++++++++++++++++++++ zaza/charm_testing/__init__.py | 0 zaza/charm_testing/collect.py | 0 zaza/charm_testing/configure.py | 34 +++++ zaza/charm_testing/deploy.py | 39 +++++ zaza/charm_testing/destroy.py | 29 ++++ zaza/charm_testing/func_test_runner.py | 33 ++++ zaza/charm_testing/prepare.py | 29 ++++ zaza/charm_testing/test.py | 34 +++++ zaza/charm_testing/utils.py | 31 ++++ zaza/charm_tests/__init__.py | 0 zaza/charm_tests/test_utils.py | 15 ++ 13 files changed, 451 insertions(+), 2 deletions(-) create mode 100644 zaza/charm_testing/README.md create mode 100644 zaza/charm_testing/__init__.py create mode 100644 zaza/charm_testing/collect.py create mode 100644 zaza/charm_testing/configure.py create mode 100755 zaza/charm_testing/deploy.py create mode 100644 zaza/charm_testing/destroy.py create mode 100644 zaza/charm_testing/func_test_runner.py create mode 100644 zaza/charm_testing/prepare.py create mode 100644 zaza/charm_testing/test.py create mode 100644 zaza/charm_testing/utils.py create mode 100644 zaza/charm_tests/__init__.py create mode 100644 zaza/charm_tests/test_utils.py diff --git a/setup.py b/setup.py index a6ff7ec..d2fde80 100644 --- a/setup.py +++ b/setup.py @@ -74,8 +74,12 @@ setup( ], entry_points={ 'console_scripts': [ - 'functest-run-tests = zaza.functests.deploy:run_tests', - 'functest-bundle-deploy = zaza.functests.deploy:deploy', + 'functest-run-suite = zaza.charm_testing.func_test_runner:main', + 'functest-deploy = zaza.charm_testing.deploy:main', + 'functest-configure = zaza.charm_testing.configure:main', + 'functest-destroy = zaza.charm_testing.destroy:main', + 'functest-prepare = zaza.charm_testing.prepare:main', + 'functest-test = zaza.charm_testing.test:main', 'current-apps = zaza.model:main', 'tempest-config = zaza.tempest_config:main', ] diff --git a/zaza/charm_testing/README.md b/zaza/charm_testing/README.md new file mode 100644 index 0000000..993115d --- /dev/null +++ b/zaza/charm_testing/README.md @@ -0,0 +1,201 @@ +# Enabling Charm Tests with zaza + +The end-to-end tests of a charm are divided into distinct pjases. Each phase +can be run in isolation and tests chared between charms. + +# Running a suite of deployments and tests + +**functest-run-suite** will read the charms tests.yaml and execute the +deployments and tests outlined there. However, each phase can be run +independantly. + +## Phases + +Charms should ship with bundles that deploy the charm with different +application versions, topologies or config options. functest-run-suite will +run through each phase listed below in order for each bundle that is to be +tested. + +### 1) Prepare + +Prepare the environment ready for a deployment. At a minimum create a model +to run the deployment in. + +To run manually: + +``` +$ functest-prepare --help +usage: functest-prepare [-h] -m MODEL_NAME + +optional arguments: + -h, --help show this help message and exit + -m MODEL_NAME, --model-name MODEL_NAME + Name of new model +``` + +### 2) Deploy + +Deploy the target bundle and wait for it to complete. **functest-run-suite** +will look at the list of bundles in the tests.yaml in the charm to determine +the bundle. + +To run manually: + +``` +$ functest-deploy --help +usage: functest-deploy [-h] -m MODEL -b BUNDLE [--no-wait] + +optional arguments: + -h, --help show this help message and exit + -m MODEL, --model MODEL + Model to deploy to + -b BUNDLE, --bundle BUNDLE + Bundle name (excluding file ext) + --no-wait Do not wait for deployment to settle +``` + +### 3) Configure + +Post-deployment configuration, for example create network, tenant, image, etc. +Any necessary post-deploy actions go here. **functest-run-suite** will look +for a list of functions that should be run in tests.yaml and execute each +in turn. + +To run manually: + +``` + functest-configure --help +usage: functest-configure [-h] [-c CONFIGFUNCS [CONFIGFUNCS ...]] + +optional arguments: + -h, --help show this help message and exit + -c CONFIGFUNCS [CONFIGFUNCS ...], --configfuncs CONFIGFUNCS [CONFIGFUNCS ...] + Space sperated list of config functions +``` + +### 4) Test + +Run tests. These maybe tests in zaza or a wrapper around another testing +framework like rally or tempest. **functest-run-suite** will look for a list +of test classes that should be run in tests.yaml and execute each in turn. + +To run manually: + +``` +usage: functest-test [-h] [-t TESTS [TESTS ...]] + +optional arguments: + -h, --help show this help message and exit + -t TESTS [TESTS ...], --tests TESTS [TESTS ...] + Space sperated list of test classes +``` + +### 5) Collect + +Collect artifacts useful for debugging any failures or infor useful for trend +anaylsis like deprecation warning or deployment time. + + +### 6) Destroy + +Destroy the model. + +``` +functest-destroy --help +usage: functest-destroy [-h] -m MODEL_NAME + +optional arguments: + -h, --help show this help message and exit + -m MODEL_NAME, --model-name MODEL_NAME + Name of model to remove +``` + +# Enabling zaza tests in a charm + + + * Add zaza in the charms test-requirements.txt + * tox.ini should include a target like: + +``` +[testenv:func35] + basepython = python3 + commands = + functest-run-suite +``` + + * Bundles which are to be used for the tests: + +``` +tests/bundles/base-xenial.yaml +tests/bundles/base-xenial-ha.yaml +tests/bundles/base-bionic.yaml +``` + + * A tests/tests.yaml file that describes the bundles to be run and + the tests +``` +charm_name: vault +tests: + - zaza.charm_tests.vault.VaultTest +configure: + - zaza.charm_tests.vault.setup.basic_setup +gate_bundles: + - base-xenial + - base-bionic +dev_bundles: + - base-xenial-ha +``` + +# Adding tests to zaza + +The setup and tests for a charm should live in zaza, this enables the code to +be shared between multiple charms. To add support for a new charm create a +directory, named after the charm, inside **zaza/charm_tests**. Within the new +directory define the tests in **tests.py** and any setup code in **setup.py** +This code can then be referenced in the charms **tests.yaml** + +eg to add support for a new congress charm create a new directory in zaza + +``` +mkdir zaza/charm_tests/congress +``` + +Add setup code into setup.py + +``` +$ cat zaza/charm_tests/congress/setup.py +def basic_setup(): + congress_client(run_special_setup) +``` + +Add test code into tests.py + +``` +class CongressTest(unittest.TestCase): + + def test_policy_create(self): + policy = congress.create_policy() + self.assertTrue(policy) +``` + +These now need to be refenced in the congress charms tests.yaml. Additional +setup is needed to run a useful congress tests, so congress' tests.yaml might +look like: + +``` +charm_name: congress +configure: + - zaza.charm_tests.nova.setup.flavor_setup + - zaza.charm_tests.nova.setup.image_setup + - zaza.charm_tests.neutron.setup.create_tenant_networks + - zaza.charm_tests.neutron.setup.create_ext_networks + - zaza.charm_tests.congress.setup.basic_setup +tests: + - zaza.charm_tests.keystone.KeystoneBasicTest + - zaza.charm_tests.congress.CongressTest +gate_bundles: + - base-xenial + - base-bionic +dev_bundles: + - base-xenial-ha +``` diff --git a/zaza/charm_testing/__init__.py b/zaza/charm_testing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/charm_testing/collect.py b/zaza/charm_testing/collect.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/charm_testing/configure.py b/zaza/charm_testing/configure.py new file mode 100644 index 0000000..78a08df --- /dev/null +++ b/zaza/charm_testing/configure.py @@ -0,0 +1,34 @@ +import argparse +import logging + +import zaza.charm_testing.utils as utils + +def run_configure_list(functions): + """Run the configure scripts as defined in the list of test classes in + series. + + :param functions: List of configure functions functions + :type tests: ['zaza.charms_tests.svc.setup', ...] + """ + for func in functions: + utils.get_class(func)() + +def configure(functions): + """Run all post-deployment configuration steps + + :param functions: List of configure functions functions + :type tests: ['zaza.charms_tests.svc.setup', ...]""" + run_configure_list(functions) + +def main(): + """Run the configuration defined by the command line args or if none were + provided read the configuration functions from the charms tests.yaml + config file""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-c','--configfuncs', nargs='+', + help='Space sperated list of config functions', + required=False) + args = parser.parse_args() + funcs = args.configfuncs or get_test_config()['configure'] + configure(funcs) diff --git a/zaza/charm_testing/deploy.py b/zaza/charm_testing/deploy.py new file mode 100755 index 0000000..c4d5792 --- /dev/null +++ b/zaza/charm_testing/deploy.py @@ -0,0 +1,39 @@ +import argparse +import logging +import subprocess +import sys + +import juju_wait + + +def deploy_bundle(bundle, model, wait=True): + """Deploy the given bundle file in the specified model + + :param bundle: Path to bundle file + :type bundle: str + :param model: Name of model to deploy bundle in + :type model: str + """ + logging.info("Deploying bundle {}".format(bundle)) + subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) + if wait: + logging.info("Waiting for environment to settle") + juju_wait.wait() + + +def main(): + """Deploy bundle""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-m','--model', + help='Model to deploy to', + required=True) + parser.add_argument('-b','--bundle', + help='Bundle name (excluding file ext)', + required=True) + parser.add_argument('--no-wait', dest='wait', + help='Do not wait for deployment to settle', + action='store_false') + parser.set_defaults(wait=True) + args = parser.parse_args() + deploy_bundle(args.bundle, args.model, wait=args.wait) diff --git a/zaza/charm_testing/destroy.py b/zaza/charm_testing/destroy.py new file mode 100644 index 0000000..89c6576 --- /dev/null +++ b/zaza/charm_testing/destroy.py @@ -0,0 +1,29 @@ +import argparse +import logging +import subprocess + +def destroy_model(model_name): + """Remove a model with the given name + + :param model: Name of model to remove + :type bundle: str + """ + logging.info("Remove model {}".format(model_name)) + subprocess.check_call(['juju', 'destroy-model', '--yes', model_name]) + +def clean_up(model_name): + """Run all steps to cleaup after a test run + + :param model: Name of model to remove + :type bundle: str + """ + destroy_model(model_name) + +def main(): + """Cleanup after test run""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-m','--model-name', help='Name of model to remove', + required=True) + args = parser.parse_args() + clean_up(args.model_name) diff --git a/zaza/charm_testing/func_test_runner.py b/zaza/charm_testing/func_test_runner.py new file mode 100644 index 0000000..39bc4a5 --- /dev/null +++ b/zaza/charm_testing/func_test_runner.py @@ -0,0 +1,33 @@ +import datetime +import logging +import os + +import zaza.charm_testing.configure as configure +import zaza.charm_testing.destroy as destroy +import zaza.charm_testing.utils as utils +import zaza.charm_testing.prepare as prepare +import zaza.charm_testing.deploy as deploy +import zaza.charm_testing.test as test + +def func_test_runner(): + """Deploy the bundles and run the tests as defined by the charms tests.yaml + """ + test_config = utils.get_charm_config() + for t in test_config['gate_bundles']: + timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + model_name = '{}{}{}'.format(test_config['charm_name'], t, timestamp) + # Prepare + prepare.add_model(model_name) + # Deploy + deploy.deploy_bundle( + os.path.join(utils.BUNDLE_DIR, '{}.yaml'.format(t)), + model_name) + # Configure + configure.run_configure_list(test_config['configure']) + # Test + test.run_test_list(test_config['tests']) + # Destroy + destroy.clean_up(model_name) + +def main(): + func_test_runner() diff --git a/zaza/charm_testing/prepare.py b/zaza/charm_testing/prepare.py new file mode 100644 index 0000000..79aabf3 --- /dev/null +++ b/zaza/charm_testing/prepare.py @@ -0,0 +1,29 @@ +import argparse +import logging +import subprocess + +def add_model(model_name): + """Add a model with the given name + + :param model: Name of model to add + :type bundle: str + """ + logging.info("Adding model {}".format(model_name)) + subprocess.check_call(['juju', 'add-model', model_name]) + +def prepare(model_name): + """Run all steps to prepare the environment before a functional test run + + :param model: Name of model to add + :type bundle: str + """ + add_model(model_name) + +def main(): + """Add a new model""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-m','--model-name', help='Name of new model', + required=True) + args = parser.parse_args() + prepare(args.model_name) diff --git a/zaza/charm_testing/test.py b/zaza/charm_testing/test.py new file mode 100644 index 0000000..58c5836 --- /dev/null +++ b/zaza/charm_testing/test.py @@ -0,0 +1,34 @@ +import argparse +import logging +import unittest + +import zaza.charm_testing.utils as utils + +def run_test_list(tests): + """Run the tests as defined in the list of test classes in series. + + :param tests: List of test class strings + :type tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] + :raises: AssertionError if test run fails + """ + for _testcase in tests: + testcase = utils.get_class(_testcase) + suite = unittest.TestLoader().loadTestsFromTestCase(testcase) + test_result = unittest.TextTestRunner(verbosity=2).run(suite) + assert test_result.wasSuccessful(), "Test run failed" + +def test(tests): + """Run all steps to execute tests against the model""" + run_test_list(tests) + +def main(): + """Run the tests defined by the command line args or if none were provided + read the tests from the charms tests.yaml config file""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-t','--tests', nargs='+', + help='Space sperated list of test classes', + required=False) + args = parser.parse_args() + tests = args.tests or get_test_config()['tests'] + test(tests) diff --git a/zaza/charm_testing/utils.py b/zaza/charm_testing/utils.py new file mode 100644 index 0000000..5494b84 --- /dev/null +++ b/zaza/charm_testing/utils.py @@ -0,0 +1,31 @@ +import importlib +import yaml + +BUNDLE_DIR = "./tests/bundles/" +DEFAULT_TEST_CONFIG = "./tests/tests.yaml" + +def get_charm_config(): + """Read the yaml test config file and return the resulting config + + :returns: Config dictionary + :rtype: dict + """ + with open(DEFAULT_TEST_CONFIG, 'r') as stream: + return yaml.load(stream) + +def get_class(class_str): + """Get the class represented by the given string + + For example, get_class('zaza.charms_tests.svc.TestSVCClass1') + returns zaza.charms_tests.svc.TestSVCClass1 + + :param class_str: Class to be returned + :type class_str: str + :returns: Test class + :rtype: class + """ + module_name = '.'.join(class_str.split('.')[:-1]) + class_name = class_str.split('.')[-1] + module = importlib.import_module(module_name) + return getattr(module, class_name) + diff --git a/zaza/charm_tests/__init__.py b/zaza/charm_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/charm_tests/test_utils.py b/zaza/charm_tests/test_utils.py new file mode 100644 index 0000000..3e81a95 --- /dev/null +++ b/zaza/charm_tests/test_utils.py @@ -0,0 +1,15 @@ +import logging +import unittest +import zaza.model + + +def skipIfNotHA(service_name): + def _skipIfNotHA_inner_1(f): + def _skipIfNotHA_inner_2(*args, **kwargs): + if len(zaza.model.get_app_ips(service_name)) > 1: + return f(*args, **kwargs) + else: + logging.warn("Skipping HA test for non-ha service {}".format(service_name)) + return _skipIfNotHA_inner_2 + + return _skipIfNotHA_inner_1 From 39d58c38be1f0a22be9accf29646afbd6cab15d2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 23 Mar 2018 17:26:00 +0000 Subject: [PATCH 14/21] Remove old functests dir --- zaza/functests/__init__.py | 0 zaza/functests/deploy.py | 99 -------------------------------------- 2 files changed, 99 deletions(-) delete mode 100644 zaza/functests/__init__.py delete mode 100755 zaza/functests/deploy.py diff --git a/zaza/functests/__init__.py b/zaza/functests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/zaza/functests/deploy.py b/zaza/functests/deploy.py deleted file mode 100755 index 0cd09d9..0000000 --- a/zaza/functests/deploy.py +++ /dev/null @@ -1,99 +0,0 @@ -import argparse -import datetime -import importlib -import logging -import os -import subprocess -import sys -import unittest -import yaml - -import juju_wait - -BUNDLE_DIR = "./tests/bundles/" -DEFAULT_TEST_CONFIG = "./tests/tests.yaml" - -def deploy_bundle(bundle, model): - """Deploy the given bundle file in the specified model - - :param bundle: Path to bundle file - :type bundle: str - :param model: Name of model to deploy bundle in - :type model: str - """ - logging.info("Deploying bundle {}".format(bundle)) - subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) - -def add_model(model_name): - """Add a model with the given name - - :param model: Name of model to add - :type bundle: str - """ - logging.info("Adding model {}".format(model_name)) - subprocess.check_call(['juju', 'add-model', model_name]) - -def get_test_class(class_str): - """Get the test class represented by the given string - - For example, get_test_class('zaza.charms_tests.svc.TestSVCClass1') - returns zaza.charms_tests.svc.TestSVCClass1 - - :param class_str: Class to be returned - :type class_str: str - :returns: Test class - :rtype: class - """ - test_module_name = '.'.join(class_str.split('.')[:-1]) - test_class_name = class_str.split('.')[-1] - test_module = importlib.import_module(test_module_name) - return getattr(test_module, test_class_name) - -def run_test_list(tests): - """Run the tests as defined in the list of test classes in series. - - :param tests: List of test class strings - :type tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] - :raises: AssertionError if test run fails - """ - for _testcase in tests: - testcase = get_test_class(_testcase) - suite = unittest.TestLoader().loadTestsFromTestCase(testcase) - test_result = unittest.TextTestRunner(verbosity=2).run(suite) - assert test_result.wasSuccessful(), "Test run failed" - -def deploy(): - """Deploy the bundles and run the tests as defined by the charms tests.yaml - """ - test_config = get_test_config() - for t in test_config['gate_bundles']: - timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') - model_name = '{}{}{}'.format(test_config['charm_name'], t, timestamp) - add_model(model_name) - deploy_bundle( - os.path.join(BUNDLE_DIR, '{}.yaml'.format(t)), - model_name) - logging.info("Waiting for environment to settle") - juju_wait.wait() - run_test_list(test_config['tests']) - -def get_test_config(): - """Read the yaml test config file and return the resulting config - - :returns: Config dictionary - :rtype: dict - """ - with open(DEFAULT_TEST_CONFIG, 'r') as stream: - return yaml.load(stream) - -def run_tests(): - """Run the tests defined by the command line args or if none were provided - read the tests from the charms tests.yaml config file""" - logging.basicConfig(level=logging.INFO) - parser = argparse.ArgumentParser() - parser.add_argument('-t','--tests', nargs='+', - help='Space sperated list of test classes', - required=False) - args = parser.parse_args() - tests = args.tests or get_test_config()['tests'] - run_test_list(tests) From 3951d6f389abd95a6a9c4b9370e64fe719f5e37f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 26 Mar 2018 08:25:42 +0000 Subject: [PATCH 15/21] Fix lint --- zaza/charm_testing/configure.py | 7 +++++-- zaza/charm_testing/deploy.py | 5 ++--- zaza/charm_testing/destroy.py | 5 ++++- zaza/charm_testing/func_test_runner.py | 3 ++- zaza/charm_testing/prepare.py | 5 ++++- zaza/charm_testing/test.py | 7 +++++-- zaza/charm_testing/utils.py | 3 ++- zaza/charm_tests/test_utils.py | 4 ++-- 8 files changed, 26 insertions(+), 13 deletions(-) diff --git a/zaza/charm_testing/configure.py b/zaza/charm_testing/configure.py index 78a08df..4d2f772 100644 --- a/zaza/charm_testing/configure.py +++ b/zaza/charm_testing/configure.py @@ -3,6 +3,7 @@ import logging import zaza.charm_testing.utils as utils + def run_configure_list(functions): """Run the configure scripts as defined in the list of test classes in series. @@ -13,6 +14,7 @@ def run_configure_list(functions): for func in functions: utils.get_class(func)() + def configure(functions): """Run all post-deployment configuration steps @@ -20,15 +22,16 @@ def configure(functions): :type tests: ['zaza.charms_tests.svc.setup', ...]""" run_configure_list(functions) + def main(): """Run the configuration defined by the command line args or if none were provided read the configuration functions from the charms tests.yaml config file""" logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser() - parser.add_argument('-c','--configfuncs', nargs='+', + parser.add_argument('-c', '--configfuncs', nargs='+', help='Space sperated list of config functions', required=False) args = parser.parse_args() - funcs = args.configfuncs or get_test_config()['configure'] + funcs = args.configfuncs or utils.get_charm_config()['configure'] configure(funcs) diff --git a/zaza/charm_testing/deploy.py b/zaza/charm_testing/deploy.py index c4d5792..ac7b430 100755 --- a/zaza/charm_testing/deploy.py +++ b/zaza/charm_testing/deploy.py @@ -1,7 +1,6 @@ import argparse import logging import subprocess -import sys import juju_wait @@ -25,10 +24,10 @@ def main(): """Deploy bundle""" logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser() - parser.add_argument('-m','--model', + parser.add_argument('-m', '--model', help='Model to deploy to', required=True) - parser.add_argument('-b','--bundle', + parser.add_argument('-b', '--bundle', help='Bundle name (excluding file ext)', required=True) parser.add_argument('--no-wait', dest='wait', diff --git a/zaza/charm_testing/destroy.py b/zaza/charm_testing/destroy.py index 89c6576..075682c 100644 --- a/zaza/charm_testing/destroy.py +++ b/zaza/charm_testing/destroy.py @@ -2,6 +2,7 @@ import argparse import logging import subprocess + def destroy_model(model_name): """Remove a model with the given name @@ -11,6 +12,7 @@ def destroy_model(model_name): logging.info("Remove model {}".format(model_name)) subprocess.check_call(['juju', 'destroy-model', '--yes', model_name]) + def clean_up(model_name): """Run all steps to cleaup after a test run @@ -19,11 +21,12 @@ def clean_up(model_name): """ destroy_model(model_name) + def main(): """Cleanup after test run""" logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser() - parser.add_argument('-m','--model-name', help='Name of model to remove', + parser.add_argument('-m', '--model-name', help='Name of model to remove', required=True) args = parser.parse_args() clean_up(args.model_name) diff --git a/zaza/charm_testing/func_test_runner.py b/zaza/charm_testing/func_test_runner.py index 39bc4a5..8ddd232 100644 --- a/zaza/charm_testing/func_test_runner.py +++ b/zaza/charm_testing/func_test_runner.py @@ -1,5 +1,4 @@ import datetime -import logging import os import zaza.charm_testing.configure as configure @@ -9,6 +8,7 @@ import zaza.charm_testing.prepare as prepare import zaza.charm_testing.deploy as deploy import zaza.charm_testing.test as test + def func_test_runner(): """Deploy the bundles and run the tests as defined by the charms tests.yaml """ @@ -29,5 +29,6 @@ def func_test_runner(): # Destroy destroy.clean_up(model_name) + def main(): func_test_runner() diff --git a/zaza/charm_testing/prepare.py b/zaza/charm_testing/prepare.py index 79aabf3..814ef64 100644 --- a/zaza/charm_testing/prepare.py +++ b/zaza/charm_testing/prepare.py @@ -2,6 +2,7 @@ import argparse import logging import subprocess + def add_model(model_name): """Add a model with the given name @@ -11,6 +12,7 @@ def add_model(model_name): logging.info("Adding model {}".format(model_name)) subprocess.check_call(['juju', 'add-model', model_name]) + def prepare(model_name): """Run all steps to prepare the environment before a functional test run @@ -19,11 +21,12 @@ def prepare(model_name): """ add_model(model_name) + def main(): """Add a new model""" logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser() - parser.add_argument('-m','--model-name', help='Name of new model', + parser.add_argument('-m', '--model-name', help='Name of new model', required=True) args = parser.parse_args() prepare(args.model_name) diff --git a/zaza/charm_testing/test.py b/zaza/charm_testing/test.py index 58c5836..353e7cd 100644 --- a/zaza/charm_testing/test.py +++ b/zaza/charm_testing/test.py @@ -4,6 +4,7 @@ import unittest import zaza.charm_testing.utils as utils + def run_test_list(tests): """Run the tests as defined in the list of test classes in series. @@ -17,18 +18,20 @@ def run_test_list(tests): test_result = unittest.TextTestRunner(verbosity=2).run(suite) assert test_result.wasSuccessful(), "Test run failed" + def test(tests): """Run all steps to execute tests against the model""" run_test_list(tests) + def main(): """Run the tests defined by the command line args or if none were provided read the tests from the charms tests.yaml config file""" logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser() - parser.add_argument('-t','--tests', nargs='+', + parser.add_argument('-t', '--tests', nargs='+', help='Space sperated list of test classes', required=False) args = parser.parse_args() - tests = args.tests or get_test_config()['tests'] + tests = args.tests or utils.get_charm_config()['tests'] test(tests) diff --git a/zaza/charm_testing/utils.py b/zaza/charm_testing/utils.py index 5494b84..26bd94d 100644 --- a/zaza/charm_testing/utils.py +++ b/zaza/charm_testing/utils.py @@ -4,6 +4,7 @@ import yaml BUNDLE_DIR = "./tests/bundles/" DEFAULT_TEST_CONFIG = "./tests/tests.yaml" + def get_charm_config(): """Read the yaml test config file and return the resulting config @@ -13,6 +14,7 @@ def get_charm_config(): with open(DEFAULT_TEST_CONFIG, 'r') as stream: return yaml.load(stream) + def get_class(class_str): """Get the class represented by the given string @@ -28,4 +30,3 @@ def get_class(class_str): class_name = class_str.split('.')[-1] module = importlib.import_module(module_name) return getattr(module, class_name) - diff --git a/zaza/charm_tests/test_utils.py b/zaza/charm_tests/test_utils.py index 3e81a95..e5e2bd7 100644 --- a/zaza/charm_tests/test_utils.py +++ b/zaza/charm_tests/test_utils.py @@ -1,5 +1,4 @@ import logging -import unittest import zaza.model @@ -9,7 +8,8 @@ def skipIfNotHA(service_name): if len(zaza.model.get_app_ips(service_name)) > 1: return f(*args, **kwargs) else: - logging.warn("Skipping HA test for non-ha service {}".format(service_name)) + logging.warn("Skipping HA test for non-ha service {}".format( + service_name)) return _skipIfNotHA_inner_2 return _skipIfNotHA_inner_1 From 588fe783572b880d31a02a744c506a29cd69a363 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 26 Mar 2018 08:33:37 +0000 Subject: [PATCH 16/21] Remove old dir --- zaza/charms_tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 zaza/charms_tests/__init__.py diff --git a/zaza/charms_tests/__init__.py b/zaza/charms_tests/__init__.py deleted file mode 100644 index e69de29..0000000 From 5d61c46a8173e2e385e23d39406aa9ff40b08bc4 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 26 Mar 2018 13:34:02 +0000 Subject: [PATCH 17/21] Fix README typos --- zaza/charm_testing/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/charm_testing/README.md b/zaza/charm_testing/README.md index 993115d..8bee2ac 100644 --- a/zaza/charm_testing/README.md +++ b/zaza/charm_testing/README.md @@ -1,15 +1,15 @@ -# Enabling Charm Tests with zaza +# Enabling Charm Functional Tests with Zaza -The end-to-end tests of a charm are divided into distinct pjases. Each phase -can be run in isolation and tests chared between charms. +The end-to-end tests of a charm are divided into distinct phases. Each phase +can be run in isolation and tests shared between charms. # Running a suite of deployments and tests **functest-run-suite** will read the charms tests.yaml and execute the deployments and tests outlined there. However, each phase can be run -independantly. +independently. -## Phases +## Charm Test Phases Charms should ship with bundles that deploy the charm with different application versions, topologies or config options. functest-run-suite will @@ -92,8 +92,8 @@ optional arguments: ### 5) Collect -Collect artifacts useful for debugging any failures or infor useful for trend -anaylsis like deprecation warning or deployment time. +Collect artifacts useful for debugging any failures or useful for trend +analysis like deprecation warning or deployment time. ### 6) Destroy From 353559528b945ec2cd4467d0b52de7f1d800d77c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 27 Mar 2018 12:33:20 +0000 Subject: [PATCH 18/21] s/charm_testing/charm_lifecycle/ and unit tests --- setup.py | 3 +- test-requirements.txt | 4 +- .../test_zaza_charm_lifecycle_configure.py | 27 ++++++ .../test_zaza_charm_lifecycle_deploy.py | 40 +++++++++ .../test_zaza_charm_lifecycle_destroy.py | 20 +++++ ...t_zaza_charm_lifecycle_func_test_runner.py | 55 +++++++++++++ .../test_zaza_charm_lifecycle_prepare.py | 20 +++++ unit_tests/test_zaza_charm_lifecycle_test.py | 36 ++++++++ unit_tests/test_zaza_charm_lifecycle_utils.py | 24 ++++++ unit_tests/utils.py | 82 +++++++++++++++++++ .../README.md | 0 .../__init__.py | 0 .../collect.py | 0 .../configure.py | 24 ++++-- .../deploy.py | 35 ++++++-- .../destroy.py | 24 ++++-- zaza/charm_lifecycle/func_test_runner.py | 38 +++++++++ .../prepare.py | 20 ++++- .../test.py | 24 ++++-- .../utils.py | 8 +- zaza/charm_testing/func_test_runner.py | 34 -------- zaza/model.py | 3 - 22 files changed, 453 insertions(+), 68 deletions(-) create mode 100644 unit_tests/test_zaza_charm_lifecycle_configure.py create mode 100644 unit_tests/test_zaza_charm_lifecycle_deploy.py create mode 100644 unit_tests/test_zaza_charm_lifecycle_destroy.py create mode 100644 unit_tests/test_zaza_charm_lifecycle_func_test_runner.py create mode 100644 unit_tests/test_zaza_charm_lifecycle_prepare.py create mode 100644 unit_tests/test_zaza_charm_lifecycle_test.py create mode 100644 unit_tests/test_zaza_charm_lifecycle_utils.py create mode 100644 unit_tests/utils.py rename zaza/{charm_testing => charm_lifecycle}/README.md (100%) rename zaza/{charm_testing => charm_lifecycle}/__init__.py (100%) rename zaza/{charm_testing => charm_lifecycle}/collect.py (100%) rename zaza/{charm_testing => charm_lifecycle}/configure.py (75%) rename zaza/{charm_testing => charm_lifecycle}/deploy.py (62%) rename zaza/{charm_testing => charm_lifecycle}/destroy.py (65%) create mode 100644 zaza/charm_lifecycle/func_test_runner.py rename zaza/{charm_testing => charm_lifecycle}/prepare.py (63%) rename zaza/{charm_testing => charm_lifecycle}/test.py (76%) rename zaza/{charm_testing => charm_lifecycle}/utils.py (79%) delete mode 100644 zaza/charm_testing/func_test_runner.py diff --git a/setup.py b/setup.py index d2fde80..ee1f491 100644 --- a/setup.py +++ b/setup.py @@ -8,9 +8,10 @@ from setuptools.command.test import test as TestCommand version = "0.0.1.dev1" install_require = [ + 'hvac', 'juju', 'juju-wait', - 'hvac' + 'PyYAML', ] tests_require = [ diff --git a/test-requirements.txt b/test-requirements.txt index ce87c70..4572d8c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,5 @@ +juju_wait +PyYAML flake8>=2.2.4,<=2.4.1 mock>=1.2 -nose>=1.3.7 \ No newline at end of file +nose>=1.3.7 diff --git a/unit_tests/test_zaza_charm_lifecycle_configure.py b/unit_tests/test_zaza_charm_lifecycle_configure.py new file mode 100644 index 0000000..8a9bf9c --- /dev/null +++ b/unit_tests/test_zaza_charm_lifecycle_configure.py @@ -0,0 +1,27 @@ +import mock + +import zaza.charm_lifecycle.configure as lc_configure +import unit_tests.utils as ut_utils + + +class TestCharmLifecycleConfigure(ut_utils.BaseTestCase): + + def test_run_configure_list(self): + self.patch_object(lc_configure.utils, 'get_class') + self.get_class.side_effect = lambda x: x + mock1 = mock.MagicMock() + mock2 = mock.MagicMock() + lc_configure.run_configure_list([mock1, mock2]) + self.assertTrue(mock1.called) + self.assertTrue(mock2.called) + + def test_configure(self): + self.patch_object(lc_configure, 'run_configure_list') + mock1 = mock.MagicMock() + mock2 = mock.MagicMock() + lc_configure.configure([mock1, mock2]) + self.run_configure_list.assert_called_once_with([mock1, mock2]) + + def test_parser(self): + args = lc_configure.parse_args(['-c', 'my.func1', 'my.func2']) + self.assertEqual(args.configfuncs, ['my.func1', 'my.func2']) diff --git a/unit_tests/test_zaza_charm_lifecycle_deploy.py b/unit_tests/test_zaza_charm_lifecycle_deploy.py new file mode 100644 index 0000000..4c764b9 --- /dev/null +++ b/unit_tests/test_zaza_charm_lifecycle_deploy.py @@ -0,0 +1,40 @@ +import zaza.charm_lifecycle.deploy as lc_deploy +import unit_tests.utils as ut_utils + + +class TestCharmLifecycleDeploy(ut_utils.BaseTestCase): + + def test_deploy_bundle(self): + self.patch_object(lc_deploy.subprocess, 'check_call') + lc_deploy.deploy_bundle('bun.yaml', 'newmodel') + self.check_call.assert_called_once_with( + ['juju', 'deploy', '-m', 'newmodel', 'bun.yaml']) + + def test_deploy(self): + self.patch_object(lc_deploy, 'deploy_bundle') + self.patch_object(lc_deploy.juju_wait, 'wait') + lc_deploy.deploy('bun.yaml', 'newmodel') + self.deploy_bundle.assert_called_once_with('bun.yaml', 'newmodel') + self.wait.assert_called_once_with() + + def test_deploy_nowait(self): + self.patch_object(lc_deploy, 'deploy_bundle') + self.patch_object(lc_deploy.juju_wait, 'wait') + lc_deploy.deploy('bun.yaml', 'newmodel', wait=False) + self.deploy_bundle.assert_called_once_with('bun.yaml', 'newmodel') + self.assertFalse(self.wait.called) + + def test_parser(self): + args = lc_deploy.parse_args([ + '-m', 'mymodel', + '-b', 'bun.yaml']) + self.assertEqual(args.model, 'mymodel') + self.assertEqual(args.bundle, 'bun.yaml') + self.assertTrue(args.wait) + + def test_parser_nowait(self): + args = lc_deploy.parse_args([ + '-m', 'mymodel', + '-b', 'bun.yaml', + '--no-wait']) + self.assertFalse(args.wait) diff --git a/unit_tests/test_zaza_charm_lifecycle_destroy.py b/unit_tests/test_zaza_charm_lifecycle_destroy.py new file mode 100644 index 0000000..55e9e83 --- /dev/null +++ b/unit_tests/test_zaza_charm_lifecycle_destroy.py @@ -0,0 +1,20 @@ +import zaza.charm_lifecycle.destroy as lc_destroy +import unit_tests.utils as ut_utils + + +class TestCharmLifecycleDestroy(ut_utils.BaseTestCase): + + def test_destroy_model(self): + self.patch_object(lc_destroy.subprocess, 'check_call') + lc_destroy.destroy_model('doomed') + self.check_call.assert_called_once_with( + ['juju', 'destroy-model', '--yes', 'doomed']) + + def test_destroy(self): + self.patch_object(lc_destroy, 'destroy_model') + lc_destroy.destroy('doomed') + self.destroy_model.assert_called_once_with('doomed') + + def test_parser(self): + args = lc_destroy.parse_args(['-m', 'doomed']) + self.assertEqual(args.model_name, 'doomed') diff --git a/unit_tests/test_zaza_charm_lifecycle_func_test_runner.py b/unit_tests/test_zaza_charm_lifecycle_func_test_runner.py new file mode 100644 index 0000000..f21a645 --- /dev/null +++ b/unit_tests/test_zaza_charm_lifecycle_func_test_runner.py @@ -0,0 +1,55 @@ +import mock + +import zaza.charm_lifecycle.func_test_runner as lc_func_test_runner +import unit_tests.utils as ut_utils + + +class TestCharmLifecycleFuncTestRunner(ut_utils.BaseTestCase): + + def test_func_test_runner(self): + self.patch_object(lc_func_test_runner.utils, 'get_charm_config') + self.patch_object(lc_func_test_runner, 'generate_model_name') + self.patch_object(lc_func_test_runner.prepare, 'prepare') + self.patch_object(lc_func_test_runner.deploy, 'deploy') + self.patch_object(lc_func_test_runner.configure, 'configure') + self.patch_object(lc_func_test_runner.test, 'test') + self.patch_object(lc_func_test_runner.destroy, 'destroy') + self.generate_model_name.return_value = 'newmodel' + self.get_charm_config.return_value = { + 'charm_name': 'mycharm', + 'gate_bundles': ['bundle1', 'bundle2'], + 'configure': [ + 'zaza.charm_tests.mycharm.setup.basic_setup' + 'zaza.charm_tests.othercharm.setup.setup'], + 'tests': [ + 'zaza.charm_tests.mycharm.tests.SmokeTest', + 'zaza.charm_tests.mycharm.tests.ComplexTest']} + lc_func_test_runner.func_test_runner() + prepare_calls = [ + mock.call('newmodel'), + mock.call('newmodel')] + deploy_calls = [ + mock.call('./tests/bundles/bundle1.yaml', 'newmodel'), + mock.call('./tests/bundles/bundle2.yaml', 'newmodel')] + configure_calls = [ + mock.call([ + 'zaza.charm_tests.mycharm.setup.basic_setup' + 'zaza.charm_tests.othercharm.setup.setup']), + mock.call([ + 'zaza.charm_tests.mycharm.setup.basic_setup' + 'zaza.charm_tests.othercharm.setup.setup'])] + test_calls = [ + mock.call([ + 'zaza.charm_tests.mycharm.tests.SmokeTest', + 'zaza.charm_tests.mycharm.tests.ComplexTest']), + mock.call([ + 'zaza.charm_tests.mycharm.tests.SmokeTest', + 'zaza.charm_tests.mycharm.tests.ComplexTest'])] + destroy_calls = [ + mock.call('newmodel'), + mock.call('newmodel')] + self.prepare.assert_has_calls(prepare_calls) + self.deploy.assert_has_calls(deploy_calls) + self.configure.assert_has_calls(configure_calls) + self.test.assert_has_calls(test_calls) + self.destroy.assert_has_calls(destroy_calls) diff --git a/unit_tests/test_zaza_charm_lifecycle_prepare.py b/unit_tests/test_zaza_charm_lifecycle_prepare.py new file mode 100644 index 0000000..92ea511 --- /dev/null +++ b/unit_tests/test_zaza_charm_lifecycle_prepare.py @@ -0,0 +1,20 @@ +import zaza.charm_lifecycle.prepare as lc_prepare +import unit_tests.utils as ut_utils + + +class TestCharmLifecyclePrepare(ut_utils.BaseTestCase): + + def test_add_model(self): + self.patch_object(lc_prepare.subprocess, 'check_call') + lc_prepare.add_model('newmodel') + self.check_call.assert_called_once_with( + ['juju', 'add-model', 'newmodel']) + + def test_prepare(self): + self.patch_object(lc_prepare, 'add_model') + lc_prepare.add_model('newmodel') + self.add_model.assert_called_once_with('newmodel') + + def test_parser(self): + args = lc_prepare.parse_args(['-m', 'newmodel']) + self.assertEqual(args.model_name, 'newmodel') diff --git a/unit_tests/test_zaza_charm_lifecycle_test.py b/unit_tests/test_zaza_charm_lifecycle_test.py new file mode 100644 index 0000000..8070602 --- /dev/null +++ b/unit_tests/test_zaza_charm_lifecycle_test.py @@ -0,0 +1,36 @@ +import mock + +import zaza.charm_lifecycle.test as lc_test +import unit_tests.utils as ut_utils + + +class TestCharmLifecycleTest(ut_utils.BaseTestCase): + + def test_run_test_list(self): + loader_mock = mock.MagicMock() + runner_mock = mock.MagicMock() + self.patch_object(lc_test.unittest, 'TestLoader') + self.patch_object(lc_test.unittest, 'TextTestRunner') + self.TestLoader.return_value = loader_mock + self.TextTestRunner.return_value = runner_mock + self.patch_object(lc_test.utils, 'get_class') + self.get_class.side_effect = lambda x: x + test_class1_mock = mock.MagicMock() + test_class2_mock = mock.MagicMock() + lc_test.run_test_list([test_class1_mock, test_class2_mock]) + loader_calls = [ + mock.call(test_class1_mock), + mock.call(test_class2_mock)] + loader_mock.loadTestsFromTestCase.assert_has_calls(loader_calls) + + def test_test(self): + self.patch_object(lc_test, 'run_test_list') + lc_test.run_test_list(['test_class1', 'test_class2']) + self.run_test_list.assert_called_once_with( + ['test_class1', 'test_class2']) + + def test_parser(self): + args = lc_test.parse_args(['-t', 'my.test_class1', 'my.test_class2']) + self.assertEqual( + args.tests, + ['my.test_class1', 'my.test_class2']) diff --git a/unit_tests/test_zaza_charm_lifecycle_utils.py b/unit_tests/test_zaza_charm_lifecycle_utils.py new file mode 100644 index 0000000..edf3647 --- /dev/null +++ b/unit_tests/test_zaza_charm_lifecycle_utils.py @@ -0,0 +1,24 @@ +import os +import tempfile +import yaml + +import zaza.charm_lifecycle.utils as lc_utils +import unit_tests.utils as ut_utils + + +class TestCharmLifecycleUtils(ut_utils.BaseTestCase): + + def test_get_charm_config(self): + f = tempfile.NamedTemporaryFile(delete=False, mode='w') + f.write(yaml.dump({'test_config': 'someconfig'})) + f.close() + charm_config = lc_utils.get_charm_config(yaml_file=f.name) + os.unlink(f.name) + self.assertEqual(charm_config, {'test_config': 'someconfig'}) + + def test_get_class(self): + self.assertEqual( + type(lc_utils.get_class('unit_tests.' + 'test_zaza_charm_lifecycle_utils.' + 'TestCharmLifecycleUtils')()), + type(self)) diff --git a/unit_tests/utils.py b/unit_tests/utils.py new file mode 100644 index 0000000..e1d0fb6 --- /dev/null +++ b/unit_tests/utils.py @@ -0,0 +1,82 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Note that the unit_tests/__init__.py also mocks out two charmhelpers imports +# that have side effects that try to apt install modules: +# sys.modules['charmhelpers.contrib.openstack.utils'] = mock.MagicMock() +# sys.modules['charmhelpers.contrib.network.ip'] = mock.MagicMock() + +import contextlib +import io +import mock +import unittest + + +@contextlib.contextmanager +def patch_open(): + '''Patch open() to allow mocking both open() itself and the file that is + yielded. + + Yields the mock for "open" and "file", respectively.''' + mock_open = mock.MagicMock(spec=open) + mock_file = mock.MagicMock(spec=io.FileIO) + + @contextlib.contextmanager + def stub_open(*args, **kwargs): + mock_open(*args, **kwargs) + yield mock_file + + with mock.patch('builtins.open', stub_open): + yield mock_open, mock_file + + +class BaseTestCase(unittest.TestCase): + + def setUp(self): + self._patches = {} + self._patches_start = {} + + def tearDown(self): + for k, v in self._patches.items(): + v.stop() + setattr(self, k, None) + self._patches = None + self._patches_start = None + + def patch_object(self, obj, attr, return_value=None, name=None, new=None, + **kwargs): + if name is None: + name = attr + if new is not None: + mocked = mock.patch.object(obj, attr, new=new, **kwargs) + else: + mocked = mock.patch.object(obj, attr, **kwargs) + self._patches[name] = mocked + started = mocked.start() + if new is None: + started.return_value = return_value + self._patches_start[name] = started + setattr(self, name, started) + + def patch(self, item, return_value=None, name=None, new=None, **kwargs): + if name is None: + raise RuntimeError("Must pass 'name' to .patch()") + if new is not None: + mocked = mock.patch(item, new=new, **kwargs) + else: + mocked = mock.patch(item, **kwargs) + self._patches[name] = mocked + started = mocked.start() + if new is None: + started.return_value = return_value + self._patches_start[name] = started + setattr(self, name, started) diff --git a/zaza/charm_testing/README.md b/zaza/charm_lifecycle/README.md similarity index 100% rename from zaza/charm_testing/README.md rename to zaza/charm_lifecycle/README.md diff --git a/zaza/charm_testing/__init__.py b/zaza/charm_lifecycle/__init__.py similarity index 100% rename from zaza/charm_testing/__init__.py rename to zaza/charm_lifecycle/__init__.py diff --git a/zaza/charm_testing/collect.py b/zaza/charm_lifecycle/collect.py similarity index 100% rename from zaza/charm_testing/collect.py rename to zaza/charm_lifecycle/collect.py diff --git a/zaza/charm_testing/configure.py b/zaza/charm_lifecycle/configure.py similarity index 75% rename from zaza/charm_testing/configure.py rename to zaza/charm_lifecycle/configure.py index 4d2f772..d93cc15 100644 --- a/zaza/charm_testing/configure.py +++ b/zaza/charm_lifecycle/configure.py @@ -1,7 +1,8 @@ import argparse import logging +import sys -import zaza.charm_testing.utils as utils +import zaza.charm_lifecycle.utils as utils def run_configure_list(functions): @@ -23,15 +24,26 @@ def configure(functions): run_configure_list(functions) +def parse_args(args): + """Parse command line arguments + + :param args: List of configure functions functions + :type list: [str1, str2,...] List of command line arguments + :returns: Parsed arguments + :rtype: Namespace + """ + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--configfuncs', nargs='+', + help='Space sperated list of config functions', + required=False) + return parser.parse_args(args) + + def main(): """Run the configuration defined by the command line args or if none were provided read the configuration functions from the charms tests.yaml config file""" logging.basicConfig(level=logging.INFO) - parser = argparse.ArgumentParser() - parser.add_argument('-c', '--configfuncs', nargs='+', - help='Space sperated list of config functions', - required=False) - args = parser.parse_args() + args = parse_args(sys.argv[1:]) funcs = args.configfuncs or utils.get_charm_config()['configure'] configure(funcs) diff --git a/zaza/charm_testing/deploy.py b/zaza/charm_lifecycle/deploy.py similarity index 62% rename from zaza/charm_testing/deploy.py rename to zaza/charm_lifecycle/deploy.py index ac7b430..39b9b71 100755 --- a/zaza/charm_testing/deploy.py +++ b/zaza/charm_lifecycle/deploy.py @@ -1,11 +1,12 @@ import argparse import logging import subprocess +import sys import juju_wait -def deploy_bundle(bundle, model, wait=True): +def deploy_bundle(bundle, model): """Deploy the given bundle file in the specified model :param bundle: Path to bundle file @@ -15,14 +16,32 @@ def deploy_bundle(bundle, model, wait=True): """ logging.info("Deploying bundle {}".format(bundle)) subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) + + +def deploy(bundle, model, wait=True): + """Run all steps to complete deployment + + :param bundle: Path to bundle file + :type bundle: str + :param model: Name of model to deploy bundle in + :type model: str + :param wait: Whether to wait until deployment completes + :type model: bool + """ + deploy_bundle(bundle, model) if wait: logging.info("Waiting for environment to settle") juju_wait.wait() -def main(): - """Deploy bundle""" - logging.basicConfig(level=logging.INFO) +def parse_args(args): + """Parse command line arguments + + :param args: List of configure functions functions + :type list: [str1, str2,...] List of command line arguments + :returns: Parsed arguments + :rtype: Namespace + """ parser = argparse.ArgumentParser() parser.add_argument('-m', '--model', help='Model to deploy to', @@ -34,5 +53,11 @@ def main(): help='Do not wait for deployment to settle', action='store_false') parser.set_defaults(wait=True) - args = parser.parse_args() + return parser.parse_args(args) + + +def main(): + """Deploy bundle""" + logging.basicConfig(level=logging.INFO) + args = parse_args(sys.argv[1:]) deploy_bundle(args.bundle, args.model, wait=args.wait) diff --git a/zaza/charm_testing/destroy.py b/zaza/charm_lifecycle/destroy.py similarity index 65% rename from zaza/charm_testing/destroy.py rename to zaza/charm_lifecycle/destroy.py index 075682c..07c2704 100644 --- a/zaza/charm_testing/destroy.py +++ b/zaza/charm_lifecycle/destroy.py @@ -1,6 +1,7 @@ import argparse import logging import subprocess +import sys def destroy_model(model_name): @@ -13,7 +14,7 @@ def destroy_model(model_name): subprocess.check_call(['juju', 'destroy-model', '--yes', model_name]) -def clean_up(model_name): +def destroy(model_name): """Run all steps to cleaup after a test run :param model: Name of model to remove @@ -22,11 +23,22 @@ def clean_up(model_name): destroy_model(model_name) -def main(): - """Cleanup after test run""" - logging.basicConfig(level=logging.INFO) +def parse_args(args): + """Parse command line arguments + + :param args: List of configure functions functions + :type list: [str1, str2,...] List of command line arguments + :returns: Parsed arguments + :rtype: Namespace + """ parser = argparse.ArgumentParser() parser.add_argument('-m', '--model-name', help='Name of model to remove', required=True) - args = parser.parse_args() - clean_up(args.model_name) + return parser.parse_args(args) + + +def main(): + """Cleanup after test run""" + logging.basicConfig(level=logging.INFO) + args = parse_args(sys.argv[1:]) + destroy(args.model_name) diff --git a/zaza/charm_lifecycle/func_test_runner.py b/zaza/charm_lifecycle/func_test_runner.py new file mode 100644 index 0000000..c7a057a --- /dev/null +++ b/zaza/charm_lifecycle/func_test_runner.py @@ -0,0 +1,38 @@ +import datetime +import os + +import zaza.charm_lifecycle.configure as configure +import zaza.charm_lifecycle.destroy as destroy +import zaza.charm_lifecycle.utils as utils +import zaza.charm_lifecycle.prepare as prepare +import zaza.charm_lifecycle.deploy as deploy +import zaza.charm_lifecycle.test as test + + +def generate_model_name(charm_name, bundle_name): + timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + return '{}{}{}'.format('charm_name', bundle_name, timestamp) + + +def func_test_runner(): + """Deploy the bundles and run the tests as defined by the charms tests.yaml + """ + test_config = utils.get_charm_config() + for t in test_config['gate_bundles']: + model_name = generate_model_name(test_config['charm_name'], t) + # Prepare + prepare.prepare(model_name) + # Deploy + deploy.deploy( + os.path.join(utils.BUNDLE_DIR, '{}.yaml'.format(t)), + model_name) + # Configure + configure.configure(test_config['configure']) + # Test + test.test(test_config['tests']) + # Destroy + destroy.destroy(model_name) + + +def main(): + func_test_runner() diff --git a/zaza/charm_testing/prepare.py b/zaza/charm_lifecycle/prepare.py similarity index 63% rename from zaza/charm_testing/prepare.py rename to zaza/charm_lifecycle/prepare.py index 814ef64..0f0e436 100644 --- a/zaza/charm_testing/prepare.py +++ b/zaza/charm_lifecycle/prepare.py @@ -1,6 +1,7 @@ import argparse import logging import subprocess +import sys def add_model(model_name): @@ -22,11 +23,22 @@ def prepare(model_name): add_model(model_name) +def parse_args(args): + """Parse command line arguments + + :param args: List of configure functions functions + :type list: [str1, str2,...] List of command line arguments + :returns: Parsed arguments + :rtype: Namespace + """ + parser = argparse.ArgumentParser() + parser.add_argument('-m', '--model-name', help='Name of model to remove', + required=True) + return parser.parse_args(args) + + def main(): """Add a new model""" logging.basicConfig(level=logging.INFO) - parser = argparse.ArgumentParser() - parser.add_argument('-m', '--model-name', help='Name of new model', - required=True) - args = parser.parse_args() + args = parse_args(sys.argv[1:]) prepare(args.model_name) diff --git a/zaza/charm_testing/test.py b/zaza/charm_lifecycle/test.py similarity index 76% rename from zaza/charm_testing/test.py rename to zaza/charm_lifecycle/test.py index 353e7cd..063ae5a 100644 --- a/zaza/charm_testing/test.py +++ b/zaza/charm_lifecycle/test.py @@ -1,8 +1,9 @@ import argparse import logging import unittest +import sys -import zaza.charm_testing.utils as utils +import zaza.charm_lifecycle.utils as utils def run_test_list(tests): @@ -24,14 +25,25 @@ def test(tests): run_test_list(tests) -def main(): - """Run the tests defined by the command line args or if none were provided - read the tests from the charms tests.yaml config file""" - logging.basicConfig(level=logging.INFO) +def parse_args(args): + """Parse command line arguments + + :param args: List of configure functions functions + :type list: [str1, str2,...] List of command line arguments + :returns: Parsed arguments + :rtype: Namespace + """ parser = argparse.ArgumentParser() parser.add_argument('-t', '--tests', nargs='+', help='Space sperated list of test classes', required=False) - args = parser.parse_args() + return parser.parse_args(args) + + +def main(): + """Run the tests defined by the command line args or if none were provided + read the tests from the charms tests.yaml config file""" + logging.basicConfig(level=logging.INFO) + args = parse_args(sys.argv[1:]) tests = args.tests or utils.get_charm_config()['tests'] test(tests) diff --git a/zaza/charm_testing/utils.py b/zaza/charm_lifecycle/utils.py similarity index 79% rename from zaza/charm_testing/utils.py rename to zaza/charm_lifecycle/utils.py index 26bd94d..25d6f1f 100644 --- a/zaza/charm_testing/utils.py +++ b/zaza/charm_lifecycle/utils.py @@ -5,13 +5,17 @@ BUNDLE_DIR = "./tests/bundles/" DEFAULT_TEST_CONFIG = "./tests/tests.yaml" -def get_charm_config(): +def get_charm_config(yaml_file=None): """Read the yaml test config file and return the resulting config + :param yaml_file: File to be read + :type yaml_file: str :returns: Config dictionary :rtype: dict """ - with open(DEFAULT_TEST_CONFIG, 'r') as stream: + if not yaml_file: + yaml_file = DEFAULT_TEST_CONFIG + with open(yaml_file, 'r') as stream: return yaml.load(stream) diff --git a/zaza/charm_testing/func_test_runner.py b/zaza/charm_testing/func_test_runner.py deleted file mode 100644 index 8ddd232..0000000 --- a/zaza/charm_testing/func_test_runner.py +++ /dev/null @@ -1,34 +0,0 @@ -import datetime -import os - -import zaza.charm_testing.configure as configure -import zaza.charm_testing.destroy as destroy -import zaza.charm_testing.utils as utils -import zaza.charm_testing.prepare as prepare -import zaza.charm_testing.deploy as deploy -import zaza.charm_testing.test as test - - -def func_test_runner(): - """Deploy the bundles and run the tests as defined by the charms tests.yaml - """ - test_config = utils.get_charm_config() - for t in test_config['gate_bundles']: - timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') - model_name = '{}{}{}'.format(test_config['charm_name'], t, timestamp) - # Prepare - prepare.add_model(model_name) - # Deploy - deploy.deploy_bundle( - os.path.join(utils.BUNDLE_DIR, '{}.yaml'.format(t)), - model_name) - # Configure - configure.run_configure_list(test_config['configure']) - # Test - test.run_test_list(test_config['tests']) - # Destroy - destroy.clean_up(model_name) - - -def main(): - func_test_runner() diff --git a/zaza/model.py b/zaza/model.py index db10db5..a03eed5 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -1,6 +1,3 @@ -import logging -import sys - from juju import loop from juju.model import Model From 98f691b24ee0468bdfd5a51fe367e9c1769e80c4 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 27 Mar 2018 12:37:36 +0000 Subject: [PATCH 19/21] Fix setup.py after rename of charm_testing --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index ee1f491..bb8d15c 100644 --- a/setup.py +++ b/setup.py @@ -75,12 +75,12 @@ setup( ], entry_points={ 'console_scripts': [ - 'functest-run-suite = zaza.charm_testing.func_test_runner:main', - 'functest-deploy = zaza.charm_testing.deploy:main', - 'functest-configure = zaza.charm_testing.configure:main', - 'functest-destroy = zaza.charm_testing.destroy:main', - 'functest-prepare = zaza.charm_testing.prepare:main', - 'functest-test = zaza.charm_testing.test:main', + 'functest-run-suite = zaza.charm_lifecycle.func_test_runner:main', + 'functest-deploy = zaza.charm_lifecycle.deploy:main', + 'functest-configure = zaza.charm_lifecycle.configure:main', + 'functest-destroy = zaza.charm_lifecycle.destroy:main', + 'functest-prepare = zaza.charm_lifecycle.prepare:main', + 'functest-test = zaza.charm_lifecycle.test:main', 'current-apps = zaza.model:main', 'tempest-config = zaza.tempest_config:main', ] From 3d190f68f3ad45bdff0c1567bac06b70359ed924 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 27 Mar 2018 12:45:59 +0000 Subject: [PATCH 20/21] Fix generate_model_name --- zaza/charm_lifecycle/func_test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/charm_lifecycle/func_test_runner.py b/zaza/charm_lifecycle/func_test_runner.py index c7a057a..9bfd67a 100644 --- a/zaza/charm_lifecycle/func_test_runner.py +++ b/zaza/charm_lifecycle/func_test_runner.py @@ -11,7 +11,7 @@ import zaza.charm_lifecycle.test as test def generate_model_name(charm_name, bundle_name): timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') - return '{}{}{}'.format('charm_name', bundle_name, timestamp) + return '{}{}{}'.format(charm_name, bundle_name, timestamp) def func_test_runner(): From a47936361b128c1a1201836e932186b997285c5a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 27 Mar 2018 13:21:28 +0000 Subject: [PATCH 21/21] Bump upper bound for flake8 to fix async lint errors --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4572d8c..7d9f11e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ juju_wait PyYAML -flake8>=2.2.4,<=2.4.1 +flake8>=2.2.4,<=3.5.0 mock>=1.2 nose>=1.3.7