From e22fa7bbf5a4f4b1c452495c51c3ceee7f825c5e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 16 Apr 2018 13:05:06 +0000 Subject: [PATCH 1/4] Add support for model specific bundle overlays This change adds support for model specific overlays from templates which are rendered at deploy time. The overlay template directory will be searched for a corresponding template (.j2). If one is found then the overlay will be rendered using environment variables matching AMULET* or ZAZA_TEMPLATE* as a context. The rendered overlay will be used on top of the specified bundle at deploy time. A default overlay is always applied "local-charm-overlay.yaml". This overlay is only used to move the location of the charm being deployed to a relative path so the bundle can use "charm: " rather than "charm: ../../../" --- setup.py | 1 + .../test_zaza_charm_lifecycle_deploy.py | 93 +++++++++++++ zaza/charm_lifecycle/README.md | 13 ++ zaza/charm_lifecycle/deploy.py | 125 +++++++++++++++++- 4 files changed, 231 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8473126..95aaf6f 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ from setuptools.command.test import test as TestCommand version = "0.0.1.dev1" install_require = [ 'hvac', + 'jinja2', 'juju', 'juju-wait', 'PyYAML', diff --git a/unit_tests/test_zaza_charm_lifecycle_deploy.py b/unit_tests/test_zaza_charm_lifecycle_deploy.py index 4c764b9..c688cbd 100644 --- a/unit_tests/test_zaza_charm_lifecycle_deploy.py +++ b/unit_tests/test_zaza_charm_lifecycle_deploy.py @@ -1,11 +1,104 @@ +import jinja2 +import mock + import zaza.charm_lifecycle.deploy as lc_deploy import unit_tests.utils as ut_utils class TestCharmLifecycleDeploy(ut_utils.BaseTestCase): + def test_is_valid_env_key(self): + self.assertTrue(lc_deploy.is_valid_env_key('AMULET_OS_VIP')) + self.assertTrue(lc_deploy.is_valid_env_key('ZAZA_TEMPLATE_VIP00')) + self.assertFalse(lc_deploy.is_valid_env_key('PATH')) + + def test_get_template_context_from_env(self): + self.patch_object(lc_deploy.os, 'environ') + self.environ.items.return_value = [ + ('AMULET_OS_VIP', '10.10.0.2'), + ('ZAZA_TEMPLATE_VIP00', '20.3.4.5'), + ('PATH', 'aa')] + self.assertEqual( + lc_deploy.get_template_context_from_env(), + {'AMULET_OS_VIP': '10.10.0.2', 'ZAZA_TEMPLATE_VIP00': '20.3.4.5'} + ) + + def test_get_overlay_template_dir(self): + self.assertEqual( + lc_deploy.get_overlay_template_dir(), + 'tests/bundles/overlays') + + def test_get_jinja2_env(self): + self.patch_object(lc_deploy, 'get_overlay_template_dir') + self.get_overlay_template_dir.return_value = 'mytemplatedir' + self.patch_object(lc_deploy.jinja2, 'Environment') + self.patch_object(lc_deploy.jinja2, 'FileSystemLoader') + jinja_env_mock = mock.MagicMock() + self.Environment.return_value = jinja_env_mock + self.assertEqual( + lc_deploy.get_jinja2_env(), + jinja_env_mock) + self.FileSystemLoader.assert_called_once_with('mytemplatedir') + + def test_get_template_name(self): + self.assertEqual( + lc_deploy.get_template_name('mybundles/mybundle.yaml'), + 'mybundle.yaml.j2') + + def test_get_template(self): + self.patch_object(lc_deploy, 'get_jinja2_env') + jinja_env_mock = mock.MagicMock() + self.get_jinja2_env.return_value = jinja_env_mock + jinja_env_mock.get_template.return_value = 'mytemplate' + self.assertEqual( + lc_deploy.get_template('mybundle.yaml'), + 'mytemplate') + + def test_get_template_missing_template(self): + self.patch_object(lc_deploy, 'get_jinja2_env') + jinja_env_mock = mock.MagicMock() + self.get_jinja2_env.return_value = jinja_env_mock + jinja_env_mock.get_template.side_effect = \ + jinja2.exceptions.TemplateNotFound(name='bob') + self.assertIsNone(lc_deploy.get_template('mybundle.yaml')) + + def test_render_overlay(self): + self.patch_object(lc_deploy, 'get_template_context_from_env') + template_mock = mock.MagicMock() + template_mock.render.return_value = 'Template contents' + self.patch_object(lc_deploy, 'get_template') + self.get_template.return_value = template_mock + m = mock.mock_open() + with mock.patch('zaza.charm_lifecycle.deploy.open', m, create=True): + lc_deploy.render_overlay('mybundle.yaml', '/tmp/') + m.assert_called_once_with('/tmp/mybundle.yaml', 'w') + handle = m() + handle.write.assert_called_once_with('Template contents') + + def test_render_overlays(self): + RESP = { + 'local-charm-overlay.yaml': '/tmp/local-charm-overlay.yaml', + 'mybundles/mybundle.yaml': '/tmp/mybundle.yaml'} + self.patch_object(lc_deploy, 'render_overlay') + self.render_overlay.side_effect = lambda x, y: RESP[x] + self.assertEqual( + lc_deploy.render_overlays('mybundles/mybundle.yaml', '/tmp'), + ['/tmp/local-charm-overlay.yaml', '/tmp/mybundle.yaml']) + + def test_render_overlays_missing(self): + RESP = { + 'local-charm-overlay.yaml': None, + 'mybundles/mybundle.yaml': '/tmp/mybundle.yaml'} + self.patch_object(lc_deploy, 'render_overlay') + self.render_overlay.side_effect = lambda x, y: RESP[x] + self.assertEqual( + lc_deploy.render_overlays('mybundles/mybundle.yaml', '/tmp'), + ['/tmp/mybundle.yaml']) + def test_deploy_bundle(self): + self.patch_object(lc_deploy, 'render_overlays') self.patch_object(lc_deploy.subprocess, 'check_call') + self.render_overlays.return_value = [] lc_deploy.deploy_bundle('bun.yaml', 'newmodel') self.check_call.assert_called_once_with( ['juju', 'deploy', '-m', 'newmodel', 'bun.yaml']) diff --git a/zaza/charm_lifecycle/README.md b/zaza/charm_lifecycle/README.md index f595320..006dc4b 100644 --- a/zaza/charm_lifecycle/README.md +++ b/zaza/charm_lifecycle/README.md @@ -39,6 +39,12 @@ Deploy the target bundle and wait for it to complete. **functest-run-suite** will look at the list of bundles in the tests.yaml in the charm to determine the bundle. +In addition to the specified bundle the overlay template directory will be +searched for a corresponding template (\.j2). If one is found +then the overlay will be rendered using environment variables matching +AMULET\* or ZAZA_TEMPLATE\* as a context. The rendered overlay will be used on +top of the specified bundle at deploy time. + To run manually: ``` @@ -134,10 +140,17 @@ commands = tests/bundles/base-xenial.yaml tests/bundles/base-xenial-ha.yaml tests/bundles/base-bionic.yaml +``` + + * Bundle overlay templates + +``` +tests/bundles/overlays/xenial-ha-mysql.yaml.j2 ``` * A tests/tests.yaml file that describes the bundles to be run and the tests + ``` charm_name: vault tests: diff --git a/zaza/charm_lifecycle/deploy.py b/zaza/charm_lifecycle/deploy.py index e120646..31a45e7 100755 --- a/zaza/charm_lifecycle/deploy.py +++ b/zaza/charm_lifecycle/deploy.py @@ -1,12 +1,131 @@ import argparse +import jinja2 import logging +import os import subprocess import sys +import tempfile import juju_wait import zaza.charm_lifecycle.utils as utils +DEFAULT_OVERLAY_TEMPLATE_DIR = 'tests/bundles/overlays' +DEFAULT_OVERLAYS = ['local-charm-overlay.yaml'] +VALID_ENVIRONMENT_KEY_PREFIXES = ['AMULET', 'ZAZA_TEMPLATE'] + + +def is_valid_env_key(key): + """Check if key is a valid environment variable name for use with template + rendering + + :param key: List of configure functions functions + :type key: str + :returns: Whether key is a valid environment variable name + :rtype: bool + """ + valid = False + for _k in VALID_ENVIRONMENT_KEY_PREFIXES: + if key.startswith(_k): + valid = True + break + return valid + + +def get_template_context_from_env(): + """Return environment variables from the current environment that can be + used for template rendering. + + :returns: Environment variable key values for use with template rendering + :rtype: dict + """ + return {k: v for k, v in os.environ.items() if is_valid_env_key(k)} + + +def get_overlay_template_dir(): + """Return the directory to look for overlay template files in. + + :returns: Overlay template file dir + :rtype: str + """ + return DEFAULT_OVERLAY_TEMPLATE_DIR + + +def get_jinja2_env(): + """Return a jinja2 environment that can be used to render templates from. + + :returns: Jinja2 template loader + :rtype: jinja2.Environment + """ + template_dir = get_overlay_template_dir() + return jinja2.Environment( + loader=jinja2.FileSystemLoader(template_dir) + ) + + +def get_template_name(target_file): + """Return the expected name of the template used to generate the + target_file + + :param target_file: File to be rendered + :type target_file: str + :returns: Name of template used to render target_file + :rtype: str + """ + return '{}.j2'.format(os.path.basename(target_file)) + + +def get_template(target_file): + """Return the jinja2 template for the given file + + :returns: Template object used to generate target_file + :rtype: jinja2.Template + """ + jinja2_env = get_jinja2_env() + try: + template = jinja2_env.get_template(get_template_name(target_file)) + except jinja2.exceptions.TemplateNotFound: + template = None + return template + + +def render_overlay(overlay_name, target_dir): + """Render the overlay template in the directory supplied + + :param overlay_name: Name of overlay to be rendered + :type overlay_name: str + :param target_dir: Directory to render overlay in + :type overlay_name: str + :returns: Path to rendered overlay + :rtype: str + """ + template = get_template(overlay_name) + rendered_template_file = os.path.join( + target_dir, + os.path.basename(overlay_name)) + with open(rendered_template_file, "w") as fh: + fh.write( + template.render(get_template_context_from_env())) + return rendered_template_file + + +def render_overlays(bundle, target_dir): + """Render the overlays for the given bundle in the directory provided + + :param bundle: Name of bundle being deployed + :type bundle: str + :param target_dir: Directory to render overlay in + :type overlay_name: str + :returns: Path to rendered overlay + :rtype: str + """ + overlays = [] + for overlay in DEFAULT_OVERLAYS + [bundle]: + rendered_overlay = render_overlay(overlay, target_dir) + if rendered_overlay: + overlays.append(rendered_overlay) + return overlays + def deploy_bundle(bundle, model): """Deploy the given bundle file in the specified model @@ -17,7 +136,11 @@ def deploy_bundle(bundle, model): :type model: str """ logging.info("Deploying bundle {}".format(bundle)) - subprocess.check_call(['juju', 'deploy', '-m', model, bundle]) + cmd = ['juju', 'deploy', '-m', model, bundle] + with tempfile.TemporaryDirectory() as tmpdirname: + for overlay in render_overlays(bundle, tmpdirname): + cmd.extend(['--overlay', overlay]) + subprocess.check_call(cmd) def deploy(bundle, model, wait=True): From 7f09add0d0252a1ea230b51d5d092c0ec87c3036 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 16 Apr 2018 14:13:23 +0000 Subject: [PATCH 2/4] Switch to OS_ as the prefix for environment variables that can be used in overlay template rendering --- unit_tests/test_zaza_charm_lifecycle_deploy.py | 8 +++++--- zaza/charm_lifecycle/README.md | 4 ++-- zaza/charm_lifecycle/deploy.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/unit_tests/test_zaza_charm_lifecycle_deploy.py b/unit_tests/test_zaza_charm_lifecycle_deploy.py index c688cbd..61688c2 100644 --- a/unit_tests/test_zaza_charm_lifecycle_deploy.py +++ b/unit_tests/test_zaza_charm_lifecycle_deploy.py @@ -8,19 +8,21 @@ import unit_tests.utils as ut_utils class TestCharmLifecycleDeploy(ut_utils.BaseTestCase): def test_is_valid_env_key(self): - self.assertTrue(lc_deploy.is_valid_env_key('AMULET_OS_VIP')) - self.assertTrue(lc_deploy.is_valid_env_key('ZAZA_TEMPLATE_VIP00')) + self.assertTrue(lc_deploy.is_valid_env_key('OS_VIP04')) + self.assertFalse(lc_deploy.is_valid_env_key('AMULET_OS_VIP')) + self.assertFalse(lc_deploy.is_valid_env_key('ZAZA_TEMPLATE_VIP00')) self.assertFalse(lc_deploy.is_valid_env_key('PATH')) def test_get_template_context_from_env(self): self.patch_object(lc_deploy.os, 'environ') self.environ.items.return_value = [ ('AMULET_OS_VIP', '10.10.0.2'), + ('OS_VIP04', '10.10.0.2'), ('ZAZA_TEMPLATE_VIP00', '20.3.4.5'), ('PATH', 'aa')] self.assertEqual( lc_deploy.get_template_context_from_env(), - {'AMULET_OS_VIP': '10.10.0.2', 'ZAZA_TEMPLATE_VIP00': '20.3.4.5'} + {'OS_VIP04': '10.10.0.2'} ) def test_get_overlay_template_dir(self): diff --git a/zaza/charm_lifecycle/README.md b/zaza/charm_lifecycle/README.md index 006dc4b..1f02c8c 100644 --- a/zaza/charm_lifecycle/README.md +++ b/zaza/charm_lifecycle/README.md @@ -41,8 +41,8 @@ the bundle. In addition to the specified bundle the overlay template directory will be searched for a corresponding template (\.j2). If one is found -then the overlay will be rendered using environment variables matching -AMULET\* or ZAZA_TEMPLATE\* as a context. The rendered overlay will be used on +then the overlay will be rendered using environment variables starting with +OS\_ as a context. The rendered overlay will be used on top of the specified bundle at deploy time. To run manually: diff --git a/zaza/charm_lifecycle/deploy.py b/zaza/charm_lifecycle/deploy.py index 31a45e7..547cf70 100755 --- a/zaza/charm_lifecycle/deploy.py +++ b/zaza/charm_lifecycle/deploy.py @@ -12,7 +12,7 @@ import zaza.charm_lifecycle.utils as utils DEFAULT_OVERLAY_TEMPLATE_DIR = 'tests/bundles/overlays' DEFAULT_OVERLAYS = ['local-charm-overlay.yaml'] -VALID_ENVIRONMENT_KEY_PREFIXES = ['AMULET', 'ZAZA_TEMPLATE'] +VALID_ENVIRONMENT_KEY_PREFIXES = ['OS_'] def is_valid_env_key(key): From 51fb79c24f1c0e3e7ca87ce9879d3b3b12b4e7be Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 16 Apr 2018 14:17:57 +0000 Subject: [PATCH 3/4] More env var refinement --- unit_tests/test_zaza_charm_lifecycle_deploy.py | 5 +++++ zaza/charm_lifecycle/deploy.py | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/unit_tests/test_zaza_charm_lifecycle_deploy.py b/unit_tests/test_zaza_charm_lifecycle_deploy.py index 61688c2..02f55b9 100644 --- a/unit_tests/test_zaza_charm_lifecycle_deploy.py +++ b/unit_tests/test_zaza_charm_lifecycle_deploy.py @@ -9,6 +9,11 @@ class TestCharmLifecycleDeploy(ut_utils.BaseTestCase): def test_is_valid_env_key(self): self.assertTrue(lc_deploy.is_valid_env_key('OS_VIP04')) + self.assertTrue(lc_deploy.is_valid_env_key('FIP_RANGE')) + self.assertTrue(lc_deploy.is_valid_env_key('GATEWAY')) + self.assertTrue(lc_deploy.is_valid_env_key('NAME_SERVER')) + self.assertTrue(lc_deploy.is_valid_env_key('NET_ID')) + self.assertTrue(lc_deploy.is_valid_env_key('VIP_RANGE')) self.assertFalse(lc_deploy.is_valid_env_key('AMULET_OS_VIP')) self.assertFalse(lc_deploy.is_valid_env_key('ZAZA_TEMPLATE_VIP00')) self.assertFalse(lc_deploy.is_valid_env_key('PATH')) diff --git a/zaza/charm_lifecycle/deploy.py b/zaza/charm_lifecycle/deploy.py index 547cf70..679589c 100755 --- a/zaza/charm_lifecycle/deploy.py +++ b/zaza/charm_lifecycle/deploy.py @@ -12,7 +12,14 @@ import zaza.charm_lifecycle.utils as utils DEFAULT_OVERLAY_TEMPLATE_DIR = 'tests/bundles/overlays' DEFAULT_OVERLAYS = ['local-charm-overlay.yaml'] -VALID_ENVIRONMENT_KEY_PREFIXES = ['OS_'] +VALID_ENVIRONMENT_KEY_PREFIXES = [ + 'FIP_RANGE', + 'GATEWAY', + 'NAME_SERVER', + 'NET_ID', + 'OS_', + 'VIP_RANGE', +] def is_valid_env_key(key): From e21dc50fddc7e915766541baf3777c083b4811b3 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 16 Apr 2018 14:21:35 +0000 Subject: [PATCH 4/4] Update readme --- zaza/charm_lifecycle/README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/zaza/charm_lifecycle/README.md b/zaza/charm_lifecycle/README.md index 1f02c8c..d96c39c 100644 --- a/zaza/charm_lifecycle/README.md +++ b/zaza/charm_lifecycle/README.md @@ -41,9 +41,17 @@ the bundle. In addition to the specified bundle the overlay template directory will be searched for a corresponding template (\.j2). If one is found -then the overlay will be rendered using environment variables starting with -OS\_ as a context. The rendered overlay will be used on -top of the specified bundle at deploy time. +then the overlay will be rendered using environment variables a specific set +of environment variables as conext. Currently these are: + + * FIP\_RANGE + * GATEWAY + * NAME\_SERVER + * NET\_ID + * OS\_\* + * VIP\_RANGE + +The rendered overlay will be used on top of the specified bundle at deploy time. To run manually: