From 056d2f41c20a5b837b1b5c53c843de7954ba2c78 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 6 Jun 2018 09:38:13 +0100 Subject: [PATCH] Split up docs and reformat * Move contents of ./zaza/charm_lifecycle/README.md into docs * Split up api docs into seperate pages * Fix prepare command help string --- doc/source/addingcharmtests.rst | 125 ++++++++++++++++ doc/source/api.rst | 55 +------ doc/source/cli-utils.rst | 5 + doc/source/exception-utils.rst | 5 + doc/source/generic-utils.rst | 5 + doc/source/index.rst | 2 + doc/source/juju-utils.rst | 5 + doc/source/model-api.rst | 5 + doc/source/openstack-utils.rst | 5 + doc/source/runningcharmtests.rst | 180 +++++++++++++++++++++++ doc/source/tls-cert-utils.rst | 5 + zaza/charm_lifecycle/README.md | 242 ------------------------------- zaza/charm_lifecycle/prepare.py | 2 +- 13 files changed, 351 insertions(+), 290 deletions(-) create mode 100644 doc/source/addingcharmtests.rst create mode 100644 doc/source/cli-utils.rst create mode 100644 doc/source/exception-utils.rst create mode 100644 doc/source/generic-utils.rst create mode 100644 doc/source/juju-utils.rst create mode 100644 doc/source/model-api.rst create mode 100644 doc/source/openstack-utils.rst create mode 100644 doc/source/runningcharmtests.rst create mode 100644 doc/source/tls-cert-utils.rst delete mode 100644 zaza/charm_lifecycle/README.md diff --git a/doc/source/addingcharmtests.rst b/doc/source/addingcharmtests.rst new file mode 100644 index 0000000..06edc68 --- /dev/null +++ b/doc/source/addingcharmtests.rst @@ -0,0 +1,125 @@ +Enabling zaza tests in a charm +============================== + +Update requirements and tox +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add zaza in the charms test-requirements.txt:: + + git+https://github.com/openstack-charmers/zaza.git#egg=zaza + + +Add targets to tox.ini should include a target like:: + + [testenv:func] + basepython = python3 + commands = + functest-run-suite --keep-model + + [testenv:func-smoke] + basepython = python3 + commands = + functest-run-suite --keep-model --smoke + +Add Bundles +~~~~~~~~~~~ + +The bundles live in tests/bundles of the built charm, eg:: + + tests/bundles/xenial.yaml + tests/bundles/xenial-ha.yaml + tests/bundles/bionic.yaml + + +The bundle may include overlay templates which are, currently, populated from +environment variables. For example the xenial-ha template needs a VIP but +the VIP will depend on the setup of the juju providor so will be different +between test environments. To accomadate this an overlay is added:: + + tests/bundles/overlays/xenial-ha.yaml.j2 + +The overlay is in jinja2 format and the variables correspond to environment +variables:: + + applications: + vault: + options: + vip: '{{ OS_VIP00 }}' + +Add tests.yaml +~~~~~~~~~~~~~~ + +A tests/tests.yaml file that describes the bundles to be run and the tests:: + + charm_name: vault + tests: + - zaza.charm_tests.vault.VaultTest + configure: + - zaza.charm_tests.vault.setup.basic_setup + gate_bundles: + - base-xenial + - base-bionic + dev_bundles: + - base-xenial-ha + smoke_bundles: + - base-bionic + +When deploying zaza will wait for the deployment to settle and for the charms +to display a workload status which indicates that they are ready. Sometimes +one or more of the applications being deployed may have a non-standard workload +status target state or message. To inform the deployment step what to +wait for an optional target\_deploy\_status stanza can be added:: + + target_deploy_status: + vault: + workload-status: blocked + workload-status-message: Vault needs to be initialized + ntp: + workload-status-message: Go for it + +Adding tests to zaza +~~~~~~~~~~~~~~~~~~~~ + +The setup and tests for a charm should live in zaza, this enables the code to +be shared between multiple charms. To add support for a new charm create a +directory, named after the charm, inside **zaza/charm_tests**. Within the new +directory define the tests in **tests.py** and any setup code in **setup.py** +This code can then be referenced in the charms **tests.yaml** + +eg to add support for a new congress charm create a new directory in zaza:: + + mkdir zaza/charm_tests/congress + +Add setup code into setup.py:: + + $ cat zaza/charm_tests/congress/setup.py + def basic_setup(): + congress_client(run_special_setup) + +Add test code into tests.py:: + + class CongressTest(unittest.TestCase): + + def test_policy_create(self): + policy = congress.create_policy() + self.assertTrue(policy) + +These now need to be refenced in the congress charms tests.yaml. Additional +setup is needed to run a useful congress tests, so congress' tests.yaml might +look like:: + + charm_name: congress + configure: + - zaza.charm_tests.nova.setup.flavor_setup + - zaza.charm_tests.nova.setup.image_setup + - zaza.charm_tests.neutron.setup.create_tenant_networks + - zaza.charm_tests.neutron.setup.create_ext_networks + - zaza.charm_tests.congress.setup.basic_setup + tests: + - zaza.charm_tests.keystone.KeystoneBasicTest + - zaza.charm_tests.congress.CongressTest + gate_bundles: + - base-xenial + - base-bionic + dev_bundles: + - base-xenial-ha diff --git a/doc/source/api.rst b/doc/source/api.rst index 79fef7b..a627eea 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -2,51 +2,12 @@ Utilities API documentation =========================== -Juju Model ----------- - -.. automodule:: zaza.model - :members: - -OpenStack Utilities -------------------- - -.. automodule:: zaza.utilities.openstack - :members: - -TLS Certificate Utilities -------------------------- - -.. automodule:: zaza.utilities.cert - :members: - -CLI Utilities -------------- - -.. automodule:: zaza.utilities.cli - :members: - -Utility Exceptions ------------------- - -.. automodule:: zaza.utilities.exceptions - :members: - -Generic Utilities ------------------ - -.. automodule:: zaza.utilities.generic - :members: - -Juju Utilities --------------- - -.. automodule:: zaza.utilities.juju - :members: - -OpenStack Versions ------------------- - -.. automodule:: zaza.utilities.os_versions - :members: +.. toctree:: + model-api + openstack-utils + tls-cert-utils + cli-utils + exception-utils + generic-utils + juju-utils diff --git a/doc/source/cli-utils.rst b/doc/source/cli-utils.rst new file mode 100644 index 0000000..4a7e8d9 --- /dev/null +++ b/doc/source/cli-utils.rst @@ -0,0 +1,5 @@ +CLI Utilities +------------- + +.. automodule:: zaza.utilities.cli + :members: diff --git a/doc/source/exception-utils.rst b/doc/source/exception-utils.rst new file mode 100644 index 0000000..51d3a0b --- /dev/null +++ b/doc/source/exception-utils.rst @@ -0,0 +1,5 @@ +Utility Exceptions +------------------ + +.. automodule:: zaza.utilities.exceptions + :members: diff --git a/doc/source/generic-utils.rst b/doc/source/generic-utils.rst new file mode 100644 index 0000000..4f29ac2 --- /dev/null +++ b/doc/source/generic-utils.rst @@ -0,0 +1,5 @@ +Generic Utilities +----------------- + +.. automodule:: zaza.utilities.generic + :members: diff --git a/doc/source/index.rst b/doc/source/index.rst index 2a07721..8ee7c7d 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -8,6 +8,8 @@ Contents: .. toctree:: :maxdepth: 2 + runningcharmtests + addingcharmtests api Indices and tables diff --git a/doc/source/juju-utils.rst b/doc/source/juju-utils.rst new file mode 100644 index 0000000..c51131f --- /dev/null +++ b/doc/source/juju-utils.rst @@ -0,0 +1,5 @@ +Juju Utilities +-------------- + +.. automodule:: zaza.utilities.juju + :members: diff --git a/doc/source/model-api.rst b/doc/source/model-api.rst new file mode 100644 index 0000000..620f623 --- /dev/null +++ b/doc/source/model-api.rst @@ -0,0 +1,5 @@ +Juju Model +---------- + +.. automodule:: zaza.model + :members: diff --git a/doc/source/openstack-utils.rst b/doc/source/openstack-utils.rst new file mode 100644 index 0000000..fc80733 --- /dev/null +++ b/doc/source/openstack-utils.rst @@ -0,0 +1,5 @@ +OpenStack Utilities +------------------- + +.. automodule:: zaza.utilities.openstack + :members: diff --git a/doc/source/runningcharmtests.rst b/doc/source/runningcharmtests.rst new file mode 100644 index 0000000..6430780 --- /dev/null +++ b/doc/source/runningcharmtests.rst @@ -0,0 +1,180 @@ +Running Charm Tests +=================== + +The end-to-end tests of a charm are divided into distinct phases. Each phase +can be run in isolation and tests shared between charms. + +Running a suite of deployments and tests +---------------------------------------- + +**functest-run-suite** will read the charms tests.yaml and execute the +deployments and tests outlined there. However, each phase can be run +independently. + +Charm Test Phases +----------------- + +Charms should ship with bundles that deploy the charm with different +application versions, topologies or config options. functest-run-suite will +run through each phase listed below in order for each bundle that is to be +tested. + +1) Prepare +~~~~~~~~~~ + +Prepare the environment ready for a deployment. At a minimum create a model +to run the deployment in. + +To run manually:: + + $ functest-prepare --help + usage: functest-prepare [-h] -m MODEL_NAME + + optional arguments: + -h, --help show this help message and exit + -m MODEL_NAME, --model-name MODEL_NAME + Name of new model + +2) Deploy +~~~~~~~~~ + +Deploy the target bundle and wait for it to complete. **functest-run-suite** +will look at the list of bundles in the tests.yaml in the charm to determine +the bundle. + +In addition to the specified bundle the overlay template directory will be +searched for a corresponding template (\.j2). If one is found +then the overlay will be rendered using environment variables a specific set +of environment variables as conext. Currently these are: + + * FIP\_RANGE + * GATEWAY + * NAME\_SERVER + * NET\_ID + * OS\_\* + * VIP\_RANGE + +The rendered overlay will be used on top of the specified bundle at deploy time. + +To run manually:: + + $ functest-deploy --help + usage: functest-deploy [-h] -m MODEL -b BUNDLE [--no-wait] + + optional arguments: + -h, --help show this help message and exit + -m MODEL, --model MODEL + Model to deploy to + -b BUNDLE, --bundle BUNDLE + Bundle name (excluding file ext) + --no-wait Do not wait for deployment to settle + + +3) Configure +~~~~~~~~~~~~ + +Post-deployment configuration, for example create network, tenant, image, etc. +Any necessary post-deploy actions go here. **functest-run-suite** will look +for a list of functions that should be run in tests.yaml and execute each +in turn. + +To run manually:: + + $ functest-configure --help + usage: functest-configure [-h] [-c CONFIGFUNCS [CONFIGFUNCS ...]] + + optional arguments: + -h, --help + show this help message and exit + -c CONFIGFUNCS, --configfuncs CONFIGFUNCS + Space sperated list of config functions + + +4) Test +~~~~~~~ + +Run tests. These maybe tests in zaza or a wrapper around another testing +framework like rally or tempest. **functest-run-suite** will look for a list +of test classes that should be run in tests.yaml and execute each in turn. + +To run manually:: + + $ functest-test --help + usage: functest-test [-h] [-t TESTS [TESTS ...]] + + optional arguments: + -h, --help show this help message and exit + -t TESTS, --tests TESTS + Space sperated list of test classes + + +5) Collect +~~~~~~~~~~ + +Collect artifacts useful for debugging any failures or useful for trend +analysis like deprecation warning or deployment time. + + +6) Destroy +~~~~~~~~~~ + +Destroy the model:: + + + $ functest-destroy --help + usage: functest-destroy [-h] -m MODEL_NAME + + optional arguments: + -h, --help show this help message and exit + -m MODEL_NAME, --model-name MODEL_NAME + Name of model to remove + +Example +------- + +First, grab the charm in question from the charm store:: + + charm pull cs:~openstack-charmers-next/vault + cd vault + +Run tests via tox +~~~~~~~~~~~~~~~~~~ + +To run all the charms functional tests:: + + tox -e func + +or just the smoke test:: + + tox -e func-smoke + +Run tests directly with functest commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Setup the virtualenv needs to be created:: + + tox -e func-noop + source .tox/func-noop/bin/activate + +All the phases can be run with a single command for a specific bundle:: + + $ functest-run-suite -b xenial-mysql + +OR each phase can be run by hand, + +Prepare phase:: + + $ functest-prepare -m testmodel + +Pick a specific bundle to test:: + + $ functest-deploy -m testmodel -b tests/bundles/xenial-mysql.yaml + +Run the configure script to prepare the environment for running tests:: + + $ functest-configure -m testmodel + +Destroy the environment:: + + $ functest-destroy -m testmodel + diff --git a/doc/source/tls-cert-utils.rst b/doc/source/tls-cert-utils.rst new file mode 100644 index 0000000..7c63217 --- /dev/null +++ b/doc/source/tls-cert-utils.rst @@ -0,0 +1,5 @@ +TLS Certificate Utilities +------------------------- + +.. automodule:: zaza.utilities.cert + :members: diff --git a/zaza/charm_lifecycle/README.md b/zaza/charm_lifecycle/README.md deleted file mode 100644 index 25514dc..0000000 --- a/zaza/charm_lifecycle/README.md +++ /dev/null @@ -1,242 +0,0 @@ -# Enabling Charm Functional Tests with Zaza - -The end-to-end tests of a charm are divided into distinct phases. Each phase -can be run in isolation and tests shared between charms. - -# Running a suite of deployments and tests - -**functest-run-suite** will read the charms tests.yaml and execute the -deployments and tests outlined there. However, each phase can be run -independently. - -## Charm Test Phases - -Charms should ship with bundles that deploy the charm with different -application versions, topologies or config options. functest-run-suite will -run through each phase listed below in order for each bundle that is to be -tested. - -### 1) Prepare - -Prepare the environment ready for a deployment. At a minimum create a model -to run the deployment in. - -To run manually: - -``` -$ functest-prepare --help -usage: functest-prepare [-h] -m MODEL_NAME - -optional arguments: - -h, --help show this help message and exit - -m MODEL_NAME, --model-name MODEL_NAME - Name of new model -``` - -### 2) Deploy - -Deploy the target bundle and wait for it to complete. **functest-run-suite** -will look at the list of bundles in the tests.yaml in the charm to determine -the bundle. - -In addition to the specified bundle the overlay template directory will be -searched for a corresponding template (\.j2). If one is found -then the overlay will be rendered using environment variables a specific set -of environment variables as conext. Currently these are: - - * FIP\_RANGE - * GATEWAY - * NAME\_SERVER - * NET\_ID - * OS\_\* - * VIP\_RANGE - -The rendered overlay will be used on top of the specified bundle at deploy time. - -To run manually: - -``` -$ functest-deploy --help -usage: functest-deploy [-h] -m MODEL -b BUNDLE [--no-wait] - -optional arguments: - -h, --help show this help message and exit - -m MODEL, --model MODEL - Model to deploy to - -b BUNDLE, --bundle BUNDLE - Bundle name (excluding file ext) - --no-wait Do not wait for deployment to settle -``` - -### 3) Configure - -Post-deployment configuration, for example create network, tenant, image, etc. -Any necessary post-deploy actions go here. **functest-run-suite** will look -for a list of functions that should be run in tests.yaml and execute each -in turn. - -To run manually: - -``` - functest-configure --help -usage: functest-configure [-h] [-c CONFIGFUNCS [CONFIGFUNCS ...]] - -optional arguments: - -h, --help show this help message and exit - -c CONFIGFUNCS [CONFIGFUNCS ...], --configfuncs CONFIGFUNCS [CONFIGFUNCS ...] - Space sperated list of config functions -``` - -### 4) Test - -Run tests. These maybe tests in zaza or a wrapper around another testing -framework like rally or tempest. **functest-run-suite** will look for a list -of test classes that should be run in tests.yaml and execute each in turn. - -To run manually: - -``` -usage: functest-test [-h] [-t TESTS [TESTS ...]] - -optional arguments: - -h, --help show this help message and exit - -t TESTS [TESTS ...], --tests TESTS [TESTS ...] - Space sperated list of test classes -``` - -### 5) Collect - -Collect artifacts useful for debugging any failures or useful for trend -analysis like deprecation warning or deployment time. - - -### 6) Destroy - -Destroy the model. - -``` -functest-destroy --help -usage: functest-destroy [-h] -m MODEL_NAME - -optional arguments: - -h, --help show this help message and exit - -m MODEL_NAME, --model-name MODEL_NAME - Name of model to remove -``` - -# Enabling zaza tests in a charm - - - * Add zaza in the charms test-requirements.txt - * tox.ini should include a target like: - -``` -[testenv:func] -basepython = python3 -commands = - functest-run-suite --keep-model - -[testenv:func-smoke] -basepython = python3 -commands = - functest-run-suite --keep-model --smoke -``` - - * Bundles which are to be used for the tests: - -``` -tests/bundles/base-xenial.yaml -tests/bundles/base-xenial-ha.yaml -tests/bundles/base-bionic.yaml -``` - - * Bundle overlay templates - -``` -tests/bundles/overlays/xenial-ha-mysql.yaml.j2 -``` - - * A tests/tests.yaml file that describes the bundles to be run and - the tests - -``` -charm_name: vault -tests: - - zaza.charm_tests.vault.VaultTest -configure: - - zaza.charm_tests.vault.setup.basic_setup -gate_bundles: - - base-xenial - - base-bionic -dev_bundles: - - base-xenial-ha -smoke_bundles: - - base-bionic -``` - - * One of the applications being deployed may have a non-standard workload - status target state or message. To inform the deployment step what to - wait for an optional target\_deploy\_status stanza can be added: - -``` -target_deploy_status: - vault: - workload-status: blocked - workload-status-message: Vault needs to be initialized - ntp: - workload-status-message: Go for it -``` - -# Adding tests to zaza - -The setup and tests for a charm should live in zaza, this enables the code to -be shared between multiple charms. To add support for a new charm create a -directory, named after the charm, inside **zaza/charm_tests**. Within the new -directory define the tests in **tests.py** and any setup code in **setup.py** -This code can then be referenced in the charms **tests.yaml** - -eg to add support for a new congress charm create a new directory in zaza - -``` -mkdir zaza/charm_tests/congress -``` - -Add setup code into setup.py - -``` -$ cat zaza/charm_tests/congress/setup.py -def basic_setup(): - congress_client(run_special_setup) -``` - -Add test code into tests.py - -``` -class CongressTest(unittest.TestCase): - - def test_policy_create(self): - policy = congress.create_policy() - self.assertTrue(policy) -``` - -These now need to be refenced in the congress charms tests.yaml. Additional -setup is needed to run a useful congress tests, so congress' tests.yaml might -look like: - -``` -charm_name: congress -configure: - - zaza.charm_tests.nova.setup.flavor_setup - - zaza.charm_tests.nova.setup.image_setup - - zaza.charm_tests.neutron.setup.create_tenant_networks - - zaza.charm_tests.neutron.setup.create_ext_networks - - zaza.charm_tests.congress.setup.basic_setup -tests: - - zaza.charm_tests.keystone.KeystoneBasicTest - - zaza.charm_tests.congress.CongressTest -gate_bundles: - - base-xenial - - base-bionic -dev_bundles: - - base-xenial-ha -``` diff --git a/zaza/charm_lifecycle/prepare.py b/zaza/charm_lifecycle/prepare.py index 00e6002..d9ffa96 100644 --- a/zaza/charm_lifecycle/prepare.py +++ b/zaza/charm_lifecycle/prepare.py @@ -48,7 +48,7 @@ def parse_args(args): :rtype: Namespace """ parser = argparse.ArgumentParser() - parser.add_argument('-m', '--model-name', help='Name of model to remove', + parser.add_argument('-m', '--model-name', help='Name of model to add', required=True) return parser.parse_args(args)