diff --git a/setup.py b/setup.py index a6ff7ec..d2fde80 100644 --- a/setup.py +++ b/setup.py @@ -74,8 +74,12 @@ setup( ], entry_points={ 'console_scripts': [ - 'functest-run-tests = zaza.functests.deploy:run_tests', - 'functest-bundle-deploy = zaza.functests.deploy:deploy', + 'functest-run-suite = zaza.charm_testing.func_test_runner:main', + 'functest-deploy = zaza.charm_testing.deploy:main', + 'functest-configure = zaza.charm_testing.configure:main', + 'functest-destroy = zaza.charm_testing.destroy:main', + 'functest-prepare = zaza.charm_testing.prepare:main', + 'functest-test = zaza.charm_testing.test:main', 'current-apps = zaza.model:main', 'tempest-config = zaza.tempest_config:main', ] diff --git a/zaza/charm_testing/README.md b/zaza/charm_testing/README.md new file mode 100644 index 0000000..993115d --- /dev/null +++ b/zaza/charm_testing/README.md @@ -0,0 +1,201 @@ +# Enabling Charm Tests with zaza + +The end-to-end tests of a charm are divided into distinct pjases. Each phase +can be run in isolation and tests chared between charms. + +# Running a suite of deployments and tests + +**functest-run-suite** will read the charms tests.yaml and execute the +deployments and tests outlined there. However, each phase can be run +independantly. + +## Phases + +Charms should ship with bundles that deploy the charm with different +application versions, topologies or config options. functest-run-suite will +run through each phase listed below in order for each bundle that is to be +tested. + +### 1) Prepare + +Prepare the environment ready for a deployment. At a minimum create a model +to run the deployment in. + +To run manually: + +``` +$ functest-prepare --help +usage: functest-prepare [-h] -m MODEL_NAME + +optional arguments: + -h, --help show this help message and exit + -m MODEL_NAME, --model-name MODEL_NAME + Name of new model +``` + +### 2) Deploy + +Deploy the target bundle and wait for it to complete. **functest-run-suite** +will look at the list of bundles in the tests.yaml in the charm to determine +the bundle. + +To run manually: + +``` +$ functest-deploy --help +usage: functest-deploy [-h] -m MODEL -b BUNDLE [--no-wait] + +optional arguments: + -h, --help show this help message and exit + -m MODEL, --model MODEL + Model to deploy to + -b BUNDLE, --bundle BUNDLE + Bundle name (excluding file ext) + --no-wait Do not wait for deployment to settle +``` + +### 3) Configure + +Post-deployment configuration, for example create network, tenant, image, etc. +Any necessary post-deploy actions go here. **functest-run-suite** will look +for a list of functions that should be run in tests.yaml and execute each +in turn. + +To run manually: + +``` + functest-configure --help +usage: functest-configure [-h] [-c CONFIGFUNCS [CONFIGFUNCS ...]] + +optional arguments: + -h, --help show this help message and exit + -c CONFIGFUNCS [CONFIGFUNCS ...], --configfuncs CONFIGFUNCS [CONFIGFUNCS ...] + Space sperated list of config functions +``` + +### 4) Test + +Run tests. These maybe tests in zaza or a wrapper around another testing +framework like rally or tempest. **functest-run-suite** will look for a list +of test classes that should be run in tests.yaml and execute each in turn. + +To run manually: + +``` +usage: functest-test [-h] [-t TESTS [TESTS ...]] + +optional arguments: + -h, --help show this help message and exit + -t TESTS [TESTS ...], --tests TESTS [TESTS ...] + Space sperated list of test classes +``` + +### 5) Collect + +Collect artifacts useful for debugging any failures or infor useful for trend +anaylsis like deprecation warning or deployment time. + + +### 6) Destroy + +Destroy the model. + +``` +functest-destroy --help +usage: functest-destroy [-h] -m MODEL_NAME + +optional arguments: + -h, --help show this help message and exit + -m MODEL_NAME, --model-name MODEL_NAME + Name of model to remove +``` + +# Enabling zaza tests in a charm + + + * Add zaza in the charms test-requirements.txt + * tox.ini should include a target like: + +``` +[testenv:func35] + basepython = python3 + commands = + functest-run-suite +``` + + * Bundles which are to be used for the tests: + +``` +tests/bundles/base-xenial.yaml +tests/bundles/base-xenial-ha.yaml +tests/bundles/base-bionic.yaml +``` + + * A tests/tests.yaml file that describes the bundles to be run and + the tests +``` +charm_name: vault +tests: + - zaza.charm_tests.vault.VaultTest +configure: + - zaza.charm_tests.vault.setup.basic_setup +gate_bundles: + - base-xenial + - base-bionic +dev_bundles: + - base-xenial-ha +``` + +# Adding tests to zaza + +The setup and tests for a charm should live in zaza, this enables the code to +be shared between multiple charms. To add support for a new charm create a +directory, named after the charm, inside **zaza/charm_tests**. Within the new +directory define the tests in **tests.py** and any setup code in **setup.py** +This code can then be referenced in the charms **tests.yaml** + +eg to add support for a new congress charm create a new directory in zaza + +``` +mkdir zaza/charm_tests/congress +``` + +Add setup code into setup.py + +``` +$ cat zaza/charm_tests/congress/setup.py +def basic_setup(): + congress_client(run_special_setup) +``` + +Add test code into tests.py + +``` +class CongressTest(unittest.TestCase): + + def test_policy_create(self): + policy = congress.create_policy() + self.assertTrue(policy) +``` + +These now need to be refenced in the congress charms tests.yaml. Additional +setup is needed to run a useful congress tests, so congress' tests.yaml might +look like: + +``` +charm_name: congress +configure: + - zaza.charm_tests.nova.setup.flavor_setup + - zaza.charm_tests.nova.setup.image_setup + - zaza.charm_tests.neutron.setup.create_tenant_networks + - zaza.charm_tests.neutron.setup.create_ext_networks + - zaza.charm_tests.congress.setup.basic_setup +tests: + - zaza.charm_tests.keystone.KeystoneBasicTest + - zaza.charm_tests.congress.CongressTest +gate_bundles: + - base-xenial + - base-bionic +dev_bundles: + - base-xenial-ha +``` diff --git a/zaza/charm_testing/__init__.py b/zaza/charm_testing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/charm_testing/collect.py b/zaza/charm_testing/collect.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/charm_testing/configure.py b/zaza/charm_testing/configure.py new file mode 100644 index 0000000..78a08df --- /dev/null +++ b/zaza/charm_testing/configure.py @@ -0,0 +1,34 @@ +import argparse +import logging + +import zaza.charm_testing.utils as utils + +def run_configure_list(functions): + """Run the configure scripts as defined in the list of test classes in + series. + + :param functions: List of configure functions functions + :type tests: ['zaza.charms_tests.svc.setup', ...] + """ + for func in functions: + utils.get_class(func)() + +def configure(functions): + """Run all post-deployment configuration steps + + :param functions: List of configure functions functions + :type tests: ['zaza.charms_tests.svc.setup', ...]""" + run_configure_list(functions) + +def main(): + """Run the configuration defined by the command line args or if none were + provided read the configuration functions from the charms tests.yaml + config file""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-c','--configfuncs', nargs='+', + help='Space sperated list of config functions', + required=False) + args = parser.parse_args() + funcs = args.configfuncs or get_test_config()['configure'] + configure(funcs) diff --git a/zaza/charm_testing/deploy.py b/zaza/charm_testing/deploy.py new file mode 100755 index 0000000..c4d5792 --- /dev/null +++ b/zaza/charm_testing/deploy.py @@ -0,0 +1,39 @@ +import argparse +import logging +import subprocess +import sys + +import juju_wait + + +def deploy_bundle(bundle, model, wait=True): + """Deploy the given bundle file in the specified model + + :param bundle: Path to bundle file + :type bundle: str + :param model: Name of model to deploy bundle in + :type model: str + """ + logging.info("Deploying bundle {}".format(bundle)) + subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) + if wait: + logging.info("Waiting for environment to settle") + juju_wait.wait() + + +def main(): + """Deploy bundle""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-m','--model', + help='Model to deploy to', + required=True) + parser.add_argument('-b','--bundle', + help='Bundle name (excluding file ext)', + required=True) + parser.add_argument('--no-wait', dest='wait', + help='Do not wait for deployment to settle', + action='store_false') + parser.set_defaults(wait=True) + args = parser.parse_args() + deploy_bundle(args.bundle, args.model, wait=args.wait) diff --git a/zaza/charm_testing/destroy.py b/zaza/charm_testing/destroy.py new file mode 100644 index 0000000..89c6576 --- /dev/null +++ b/zaza/charm_testing/destroy.py @@ -0,0 +1,29 @@ +import argparse +import logging +import subprocess + +def destroy_model(model_name): + """Remove a model with the given name + + :param model: Name of model to remove + :type bundle: str + """ + logging.info("Remove model {}".format(model_name)) + subprocess.check_call(['juju', 'destroy-model', '--yes', model_name]) + +def clean_up(model_name): + """Run all steps to cleaup after a test run + + :param model: Name of model to remove + :type bundle: str + """ + destroy_model(model_name) + +def main(): + """Cleanup after test run""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-m','--model-name', help='Name of model to remove', + required=True) + args = parser.parse_args() + clean_up(args.model_name) diff --git a/zaza/charm_testing/func_test_runner.py b/zaza/charm_testing/func_test_runner.py new file mode 100644 index 0000000..39bc4a5 --- /dev/null +++ b/zaza/charm_testing/func_test_runner.py @@ -0,0 +1,33 @@ +import datetime +import logging +import os + +import zaza.charm_testing.configure as configure +import zaza.charm_testing.destroy as destroy +import zaza.charm_testing.utils as utils +import zaza.charm_testing.prepare as prepare +import zaza.charm_testing.deploy as deploy +import zaza.charm_testing.test as test + +def func_test_runner(): + """Deploy the bundles and run the tests as defined by the charms tests.yaml + """ + test_config = utils.get_charm_config() + for t in test_config['gate_bundles']: + timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + model_name = '{}{}{}'.format(test_config['charm_name'], t, timestamp) + # Prepare + prepare.add_model(model_name) + # Deploy + deploy.deploy_bundle( + os.path.join(utils.BUNDLE_DIR, '{}.yaml'.format(t)), + model_name) + # Configure + configure.run_configure_list(test_config['configure']) + # Test + test.run_test_list(test_config['tests']) + # Destroy + destroy.clean_up(model_name) + +def main(): + func_test_runner() diff --git a/zaza/charm_testing/prepare.py b/zaza/charm_testing/prepare.py new file mode 100644 index 0000000..79aabf3 --- /dev/null +++ b/zaza/charm_testing/prepare.py @@ -0,0 +1,29 @@ +import argparse +import logging +import subprocess + +def add_model(model_name): + """Add a model with the given name + + :param model: Name of model to add + :type bundle: str + """ + logging.info("Adding model {}".format(model_name)) + subprocess.check_call(['juju', 'add-model', model_name]) + +def prepare(model_name): + """Run all steps to prepare the environment before a functional test run + + :param model: Name of model to add + :type bundle: str + """ + add_model(model_name) + +def main(): + """Add a new model""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-m','--model-name', help='Name of new model', + required=True) + args = parser.parse_args() + prepare(args.model_name) diff --git a/zaza/charm_testing/test.py b/zaza/charm_testing/test.py new file mode 100644 index 0000000..58c5836 --- /dev/null +++ b/zaza/charm_testing/test.py @@ -0,0 +1,34 @@ +import argparse +import logging +import unittest + +import zaza.charm_testing.utils as utils + +def run_test_list(tests): + """Run the tests as defined in the list of test classes in series. + + :param tests: List of test class strings + :type tests: ['zaza.charms_tests.svc.TestSVCClass1', ...] + :raises: AssertionError if test run fails + """ + for _testcase in tests: + testcase = utils.get_class(_testcase) + suite = unittest.TestLoader().loadTestsFromTestCase(testcase) + test_result = unittest.TextTestRunner(verbosity=2).run(suite) + assert test_result.wasSuccessful(), "Test run failed" + +def test(tests): + """Run all steps to execute tests against the model""" + run_test_list(tests) + +def main(): + """Run the tests defined by the command line args or if none were provided + read the tests from the charms tests.yaml config file""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('-t','--tests', nargs='+', + help='Space sperated list of test classes', + required=False) + args = parser.parse_args() + tests = args.tests or get_test_config()['tests'] + test(tests) diff --git a/zaza/charm_testing/utils.py b/zaza/charm_testing/utils.py new file mode 100644 index 0000000..5494b84 --- /dev/null +++ b/zaza/charm_testing/utils.py @@ -0,0 +1,31 @@ +import importlib +import yaml + +BUNDLE_DIR = "./tests/bundles/" +DEFAULT_TEST_CONFIG = "./tests/tests.yaml" + +def get_charm_config(): + """Read the yaml test config file and return the resulting config + + :returns: Config dictionary + :rtype: dict + """ + with open(DEFAULT_TEST_CONFIG, 'r') as stream: + return yaml.load(stream) + +def get_class(class_str): + """Get the class represented by the given string + + For example, get_class('zaza.charms_tests.svc.TestSVCClass1') + returns zaza.charms_tests.svc.TestSVCClass1 + + :param class_str: Class to be returned + :type class_str: str + :returns: Test class + :rtype: class + """ + module_name = '.'.join(class_str.split('.')[:-1]) + class_name = class_str.split('.')[-1] + module = importlib.import_module(module_name) + return getattr(module, class_name) + diff --git a/zaza/charm_tests/__init__.py b/zaza/charm_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zaza/charm_tests/test_utils.py b/zaza/charm_tests/test_utils.py new file mode 100644 index 0000000..3e81a95 --- /dev/null +++ b/zaza/charm_tests/test_utils.py @@ -0,0 +1,15 @@ +import logging +import unittest +import zaza.model + + +def skipIfNotHA(service_name): + def _skipIfNotHA_inner_1(f): + def _skipIfNotHA_inner_2(*args, **kwargs): + if len(zaza.model.get_app_ips(service_name)) > 1: + return f(*args, **kwargs) + else: + logging.warn("Skipping HA test for non-ha service {}".format(service_name)) + return _skipIfNotHA_inner_2 + + return _skipIfNotHA_inner_1