diff --git a/setup.py b/setup.py index 8473126..95aaf6f 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ from setuptools.command.test import test as TestCommand version = "0.0.1.dev1" install_require = [ 'hvac', + 'jinja2', 'juju', 'juju-wait', 'PyYAML', diff --git a/unit_tests/test_zaza_charm_lifecycle_deploy.py b/unit_tests/test_zaza_charm_lifecycle_deploy.py index 4c764b9..02f55b9 100644 --- a/unit_tests/test_zaza_charm_lifecycle_deploy.py +++ b/unit_tests/test_zaza_charm_lifecycle_deploy.py @@ -1,11 +1,111 @@ +import jinja2 +import mock + import zaza.charm_lifecycle.deploy as lc_deploy import unit_tests.utils as ut_utils class TestCharmLifecycleDeploy(ut_utils.BaseTestCase): + def test_is_valid_env_key(self): + self.assertTrue(lc_deploy.is_valid_env_key('OS_VIP04')) + self.assertTrue(lc_deploy.is_valid_env_key('FIP_RANGE')) + self.assertTrue(lc_deploy.is_valid_env_key('GATEWAY')) + self.assertTrue(lc_deploy.is_valid_env_key('NAME_SERVER')) + self.assertTrue(lc_deploy.is_valid_env_key('NET_ID')) + self.assertTrue(lc_deploy.is_valid_env_key('VIP_RANGE')) + self.assertFalse(lc_deploy.is_valid_env_key('AMULET_OS_VIP')) + self.assertFalse(lc_deploy.is_valid_env_key('ZAZA_TEMPLATE_VIP00')) + self.assertFalse(lc_deploy.is_valid_env_key('PATH')) + + def test_get_template_context_from_env(self): + self.patch_object(lc_deploy.os, 'environ') + self.environ.items.return_value = [ + ('AMULET_OS_VIP', '10.10.0.2'), + ('OS_VIP04', '10.10.0.2'), + ('ZAZA_TEMPLATE_VIP00', '20.3.4.5'), + ('PATH', 'aa')] + self.assertEqual( + lc_deploy.get_template_context_from_env(), + {'OS_VIP04': '10.10.0.2'} + ) + + def test_get_overlay_template_dir(self): + self.assertEqual( + lc_deploy.get_overlay_template_dir(), + 'tests/bundles/overlays') + + def test_get_jinja2_env(self): + self.patch_object(lc_deploy, 'get_overlay_template_dir') + self.get_overlay_template_dir.return_value = 'mytemplatedir' + self.patch_object(lc_deploy.jinja2, 'Environment') + self.patch_object(lc_deploy.jinja2, 'FileSystemLoader') + jinja_env_mock = mock.MagicMock() + self.Environment.return_value = jinja_env_mock + self.assertEqual( + lc_deploy.get_jinja2_env(), + jinja_env_mock) + self.FileSystemLoader.assert_called_once_with('mytemplatedir') + + def test_get_template_name(self): + self.assertEqual( + lc_deploy.get_template_name('mybundles/mybundle.yaml'), + 'mybundle.yaml.j2') + + def test_get_template(self): + self.patch_object(lc_deploy, 'get_jinja2_env') + jinja_env_mock = mock.MagicMock() + self.get_jinja2_env.return_value = jinja_env_mock + jinja_env_mock.get_template.return_value = 'mytemplate' + self.assertEqual( + lc_deploy.get_template('mybundle.yaml'), + 'mytemplate') + + def test_get_template_missing_template(self): + self.patch_object(lc_deploy, 'get_jinja2_env') + jinja_env_mock = mock.MagicMock() + self.get_jinja2_env.return_value = jinja_env_mock + jinja_env_mock.get_template.side_effect = \ + jinja2.exceptions.TemplateNotFound(name='bob') + self.assertIsNone(lc_deploy.get_template('mybundle.yaml')) + + def test_render_overlay(self): + self.patch_object(lc_deploy, 'get_template_context_from_env') + template_mock = mock.MagicMock() + template_mock.render.return_value = 'Template contents' + self.patch_object(lc_deploy, 'get_template') + self.get_template.return_value = template_mock + m = mock.mock_open() + with mock.patch('zaza.charm_lifecycle.deploy.open', m, create=True): + lc_deploy.render_overlay('mybundle.yaml', '/tmp/') + m.assert_called_once_with('/tmp/mybundle.yaml', 'w') + handle = m() + handle.write.assert_called_once_with('Template contents') + + def test_render_overlays(self): + RESP = { + 'local-charm-overlay.yaml': '/tmp/local-charm-overlay.yaml', + 'mybundles/mybundle.yaml': '/tmp/mybundle.yaml'} + self.patch_object(lc_deploy, 'render_overlay') + self.render_overlay.side_effect = lambda x, y: RESP[x] + self.assertEqual( + lc_deploy.render_overlays('mybundles/mybundle.yaml', '/tmp'), + ['/tmp/local-charm-overlay.yaml', '/tmp/mybundle.yaml']) + + def test_render_overlays_missing(self): + RESP = { + 'local-charm-overlay.yaml': None, + 'mybundles/mybundle.yaml': '/tmp/mybundle.yaml'} + self.patch_object(lc_deploy, 'render_overlay') + self.render_overlay.side_effect = lambda x, y: RESP[x] + self.assertEqual( + lc_deploy.render_overlays('mybundles/mybundle.yaml', '/tmp'), + ['/tmp/mybundle.yaml']) + def test_deploy_bundle(self): + self.patch_object(lc_deploy, 'render_overlays') self.patch_object(lc_deploy.subprocess, 'check_call') + self.render_overlays.return_value = [] lc_deploy.deploy_bundle('bun.yaml', 'newmodel') self.check_call.assert_called_once_with( ['juju', 'deploy', '-m', 'newmodel', 'bun.yaml']) diff --git a/zaza/charm_lifecycle/README.md b/zaza/charm_lifecycle/README.md index f595320..d96c39c 100644 --- a/zaza/charm_lifecycle/README.md +++ b/zaza/charm_lifecycle/README.md @@ -39,6 +39,20 @@ Deploy the target bundle and wait for it to complete. **functest-run-suite** will look at the list of bundles in the tests.yaml in the charm to determine the bundle. +In addition to the specified bundle the overlay template directory will be +searched for a corresponding template (\.j2). If one is found +then the overlay will be rendered using environment variables a specific set +of environment variables as conext. Currently these are: + + * FIP\_RANGE + * GATEWAY + * NAME\_SERVER + * NET\_ID + * OS\_\* + * VIP\_RANGE + +The rendered overlay will be used on top of the specified bundle at deploy time. + To run manually: ``` @@ -134,10 +148,17 @@ commands = tests/bundles/base-xenial.yaml tests/bundles/base-xenial-ha.yaml tests/bundles/base-bionic.yaml +``` + + * Bundle overlay templates + +``` +tests/bundles/overlays/xenial-ha-mysql.yaml.j2 ``` * A tests/tests.yaml file that describes the bundles to be run and the tests + ``` charm_name: vault tests: diff --git a/zaza/charm_lifecycle/deploy.py b/zaza/charm_lifecycle/deploy.py index e120646..679589c 100755 --- a/zaza/charm_lifecycle/deploy.py +++ b/zaza/charm_lifecycle/deploy.py @@ -1,12 +1,138 @@ import argparse +import jinja2 import logging +import os import subprocess import sys +import tempfile import juju_wait import zaza.charm_lifecycle.utils as utils +DEFAULT_OVERLAY_TEMPLATE_DIR = 'tests/bundles/overlays' +DEFAULT_OVERLAYS = ['local-charm-overlay.yaml'] +VALID_ENVIRONMENT_KEY_PREFIXES = [ + 'FIP_RANGE', + 'GATEWAY', + 'NAME_SERVER', + 'NET_ID', + 'OS_', + 'VIP_RANGE', +] + + +def is_valid_env_key(key): + """Check if key is a valid environment variable name for use with template + rendering + + :param key: List of configure functions functions + :type key: str + :returns: Whether key is a valid environment variable name + :rtype: bool + """ + valid = False + for _k in VALID_ENVIRONMENT_KEY_PREFIXES: + if key.startswith(_k): + valid = True + break + return valid + + +def get_template_context_from_env(): + """Return environment variables from the current environment that can be + used for template rendering. + + :returns: Environment variable key values for use with template rendering + :rtype: dict + """ + return {k: v for k, v in os.environ.items() if is_valid_env_key(k)} + + +def get_overlay_template_dir(): + """Return the directory to look for overlay template files in. + + :returns: Overlay template file dir + :rtype: str + """ + return DEFAULT_OVERLAY_TEMPLATE_DIR + + +def get_jinja2_env(): + """Return a jinja2 environment that can be used to render templates from. + + :returns: Jinja2 template loader + :rtype: jinja2.Environment + """ + template_dir = get_overlay_template_dir() + return jinja2.Environment( + loader=jinja2.FileSystemLoader(template_dir) + ) + + +def get_template_name(target_file): + """Return the expected name of the template used to generate the + target_file + + :param target_file: File to be rendered + :type target_file: str + :returns: Name of template used to render target_file + :rtype: str + """ + return '{}.j2'.format(os.path.basename(target_file)) + + +def get_template(target_file): + """Return the jinja2 template for the given file + + :returns: Template object used to generate target_file + :rtype: jinja2.Template + """ + jinja2_env = get_jinja2_env() + try: + template = jinja2_env.get_template(get_template_name(target_file)) + except jinja2.exceptions.TemplateNotFound: + template = None + return template + + +def render_overlay(overlay_name, target_dir): + """Render the overlay template in the directory supplied + + :param overlay_name: Name of overlay to be rendered + :type overlay_name: str + :param target_dir: Directory to render overlay in + :type overlay_name: str + :returns: Path to rendered overlay + :rtype: str + """ + template = get_template(overlay_name) + rendered_template_file = os.path.join( + target_dir, + os.path.basename(overlay_name)) + with open(rendered_template_file, "w") as fh: + fh.write( + template.render(get_template_context_from_env())) + return rendered_template_file + + +def render_overlays(bundle, target_dir): + """Render the overlays for the given bundle in the directory provided + + :param bundle: Name of bundle being deployed + :type bundle: str + :param target_dir: Directory to render overlay in + :type overlay_name: str + :returns: Path to rendered overlay + :rtype: str + """ + overlays = [] + for overlay in DEFAULT_OVERLAYS + [bundle]: + rendered_overlay = render_overlay(overlay, target_dir) + if rendered_overlay: + overlays.append(rendered_overlay) + return overlays + def deploy_bundle(bundle, model): """Deploy the given bundle file in the specified model @@ -17,7 +143,11 @@ def deploy_bundle(bundle, model): :type model: str """ logging.info("Deploying bundle {}".format(bundle)) - subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) + cmd = ['juju', 'deploy', '-m', model, bundle] + with tempfile.TemporaryDirectory() as tmpdirname: + for overlay in render_overlays(bundle, tmpdirname): + cmd.extend(['--overlay', overlay]) + subprocess.check_call(cmd) def deploy(bundle, model, wait=True):