Merge pull request #20 from gnuoy/dynamic-bundles

Add support for model specific bundle overlays

setup.py

@@ -9,6 +9,7 @@ from setuptools.command.test import test as TestCommand
 version = "0.0.1.dev1"
 install_require = [
     'hvac',
+    'jinja2',
     'juju',
     'juju-wait',
     'PyYAML',

unit_tests/test_zaza_charm_lifecycle_deploy.py

@@ -1,11 +1,111 @@
+import jinja2
+import mock
+
 import zaza.charm_lifecycle.deploy as lc_deploy
 import unit_tests.utils as ut_utils
 
 
 class TestCharmLifecycleDeploy(ut_utils.BaseTestCase):
 
+    def test_is_valid_env_key(self):
+        self.assertTrue(lc_deploy.is_valid_env_key('OS_VIP04'))
+        self.assertTrue(lc_deploy.is_valid_env_key('FIP_RANGE'))
+        self.assertTrue(lc_deploy.is_valid_env_key('GATEWAY'))
+        self.assertTrue(lc_deploy.is_valid_env_key('NAME_SERVER'))
+        self.assertTrue(lc_deploy.is_valid_env_key('NET_ID'))
+        self.assertTrue(lc_deploy.is_valid_env_key('VIP_RANGE'))
+        self.assertFalse(lc_deploy.is_valid_env_key('AMULET_OS_VIP'))
+        self.assertFalse(lc_deploy.is_valid_env_key('ZAZA_TEMPLATE_VIP00'))
+        self.assertFalse(lc_deploy.is_valid_env_key('PATH'))
+
+    def test_get_template_context_from_env(self):
+        self.patch_object(lc_deploy.os, 'environ')
+        self.environ.items.return_value = [
+            ('AMULET_OS_VIP', '10.10.0.2'),
+            ('OS_VIP04', '10.10.0.2'),
+            ('ZAZA_TEMPLATE_VIP00', '20.3.4.5'),
+            ('PATH', 'aa')]
+        self.assertEqual(
+            lc_deploy.get_template_context_from_env(),
+            {'OS_VIP04': '10.10.0.2'}
+        )
+
+    def test_get_overlay_template_dir(self):
+        self.assertEqual(
+            lc_deploy.get_overlay_template_dir(),
+            'tests/bundles/overlays')
+
+    def test_get_jinja2_env(self):
+        self.patch_object(lc_deploy, 'get_overlay_template_dir')
+        self.get_overlay_template_dir.return_value = 'mytemplatedir'
+        self.patch_object(lc_deploy.jinja2, 'Environment')
+        self.patch_object(lc_deploy.jinja2, 'FileSystemLoader')
+        jinja_env_mock = mock.MagicMock()
+        self.Environment.return_value = jinja_env_mock
+        self.assertEqual(
+            lc_deploy.get_jinja2_env(),
+            jinja_env_mock)
+        self.FileSystemLoader.assert_called_once_with('mytemplatedir')
+
+    def test_get_template_name(self):
+        self.assertEqual(
+            lc_deploy.get_template_name('mybundles/mybundle.yaml'),
+            'mybundle.yaml.j2')
+
+    def test_get_template(self):
+        self.patch_object(lc_deploy, 'get_jinja2_env')
+        jinja_env_mock = mock.MagicMock()
+        self.get_jinja2_env.return_value = jinja_env_mock
+        jinja_env_mock.get_template.return_value = 'mytemplate'
+        self.assertEqual(
+            lc_deploy.get_template('mybundle.yaml'),
+            'mytemplate')
+
+    def test_get_template_missing_template(self):
+        self.patch_object(lc_deploy, 'get_jinja2_env')
+        jinja_env_mock = mock.MagicMock()
+        self.get_jinja2_env.return_value = jinja_env_mock
+        jinja_env_mock.get_template.side_effect = \
+            jinja2.exceptions.TemplateNotFound(name='bob')
+        self.assertIsNone(lc_deploy.get_template('mybundle.yaml'))
+
+    def test_render_overlay(self):
+        self.patch_object(lc_deploy, 'get_template_context_from_env')
+        template_mock = mock.MagicMock()
+        template_mock.render.return_value = 'Template contents'
+        self.patch_object(lc_deploy, 'get_template')
+        self.get_template.return_value = template_mock
+        m = mock.mock_open()
+        with mock.patch('zaza.charm_lifecycle.deploy.open', m, create=True):
+            lc_deploy.render_overlay('mybundle.yaml', '/tmp/')
+        m.assert_called_once_with('/tmp/mybundle.yaml', 'w')
+        handle = m()
+        handle.write.assert_called_once_with('Template contents')
+
+    def test_render_overlays(self):
+        RESP = {
+            'local-charm-overlay.yaml': '/tmp/local-charm-overlay.yaml',
+            'mybundles/mybundle.yaml': '/tmp/mybundle.yaml'}
+        self.patch_object(lc_deploy, 'render_overlay')
+        self.render_overlay.side_effect = lambda x, y: RESP[x]
+        self.assertEqual(
+            lc_deploy.render_overlays('mybundles/mybundle.yaml', '/tmp'),
+            ['/tmp/local-charm-overlay.yaml', '/tmp/mybundle.yaml'])
+
+    def test_render_overlays_missing(self):
+        RESP = {
+            'local-charm-overlay.yaml': None,
+            'mybundles/mybundle.yaml': '/tmp/mybundle.yaml'}
+        self.patch_object(lc_deploy, 'render_overlay')
+        self.render_overlay.side_effect = lambda x, y: RESP[x]
+        self.assertEqual(
+            lc_deploy.render_overlays('mybundles/mybundle.yaml', '/tmp'),
+            ['/tmp/mybundle.yaml'])
+
     def test_deploy_bundle(self):
+        self.patch_object(lc_deploy, 'render_overlays')
         self.patch_object(lc_deploy.subprocess, 'check_call')
+        self.render_overlays.return_value = []
         lc_deploy.deploy_bundle('bun.yaml', 'newmodel')
         self.check_call.assert_called_once_with(
             ['juju', 'deploy', '-m', 'newmodel', 'bun.yaml'])

zaza/charm_lifecycle/README.md

@@ -39,6 +39,20 @@ Deploy the target bundle and wait for it to complete. **functest-run-suite**
 will look at the list of bundles in the tests.yaml in the charm to determine
 the bundle.
 
+In addition to the specified bundle the overlay template directory will be
+searched for a corresponding template (\<bundle\_name\>.j2). If one is found
+then the overlay will be rendered using environment variables a specific set
+of environment variables as conext. Currently these are:
+
+ * FIP\_RANGE
+ * GATEWAY
+ * NAME\_SERVER
+ * NET\_ID
+ * OS\_\*
+ * VIP\_RANGE
+
+The rendered overlay will be used on top of the specified bundle at deploy time.
+
 To run manually:
 
 ```
@@ -134,10 +148,17 @@ commands =
 tests/bundles/base-xenial.yaml
 tests/bundles/base-xenial-ha.yaml
 tests/bundles/base-bionic.yaml
+```
+
+ * Bundle overlay templates
+
+```
+tests/bundles/overlays/xenial-ha-mysql.yaml.j2
 ```
 
  * A tests/tests.yaml file that describes the bundles to be run and
    the tests
+
 ```
 charm_name: vault
 tests:

zaza/charm_lifecycle/deploy.py

@@ -1,12 +1,138 @@
 import argparse
+import jinja2
 import logging
+import os
 import subprocess
 import sys
+import tempfile
 
 import juju_wait
 
 import zaza.charm_lifecycle.utils as utils
 
+DEFAULT_OVERLAY_TEMPLATE_DIR = 'tests/bundles/overlays'
+DEFAULT_OVERLAYS = ['local-charm-overlay.yaml']
+VALID_ENVIRONMENT_KEY_PREFIXES = [
+    'FIP_RANGE',
+    'GATEWAY',
+    'NAME_SERVER',
+    'NET_ID',
+    'OS_',
+    'VIP_RANGE',
+]
+
+
+def is_valid_env_key(key):
+    """Check if key is a valid environment variable name for use with template
+       rendering
+
+    :param key: List of configure functions functions
+    :type key: str
+    :returns: Whether key is a valid environment variable name
+    :rtype: bool
+    """
+    valid = False
+    for _k in VALID_ENVIRONMENT_KEY_PREFIXES:
+        if key.startswith(_k):
+            valid = True
+            break
+    return valid
+
+
+def get_template_context_from_env():
+    """Return environment variables from the current environment that can be
+       used for template rendering.
+
+    :returns: Environment variable key values for use with template rendering
+    :rtype: dict
+    """
+    return {k: v for k, v in os.environ.items() if is_valid_env_key(k)}
+
+
+def get_overlay_template_dir():
+    """Return the directory to look for overlay template files in.
+
+    :returns: Overlay template file dir
+    :rtype: str
+    """
+    return DEFAULT_OVERLAY_TEMPLATE_DIR
+
+
+def get_jinja2_env():
+    """Return a jinja2 environment that can be used to render templates from.
+
+    :returns: Jinja2 template loader
+    :rtype: jinja2.Environment
+    """
+    template_dir = get_overlay_template_dir()
+    return jinja2.Environment(
+        loader=jinja2.FileSystemLoader(template_dir)
+    )
+
+
+def get_template_name(target_file):
+    """Return the expected name of the template used to generate the
+       target_file
+
+    :param target_file: File to be rendered
+    :type target_file: str
+    :returns: Name of template used to render target_file
+    :rtype: str
+    """
+    return '{}.j2'.format(os.path.basename(target_file))
+
+
+def get_template(target_file):
+    """Return the jinja2 template for the given file
+
+    :returns: Template object used to generate target_file
+    :rtype: jinja2.Template
+    """
+    jinja2_env = get_jinja2_env()
+    try:
+        template = jinja2_env.get_template(get_template_name(target_file))
+    except jinja2.exceptions.TemplateNotFound:
+        template = None
+    return template
+
+
+def render_overlay(overlay_name, target_dir):
+    """Render the overlay template in the directory supplied
+
+    :param overlay_name: Name of overlay to be rendered
+    :type overlay_name: str
+    :param target_dir: Directory to render overlay in
+    :type overlay_name: str
+    :returns: Path to rendered overlay
+    :rtype: str
+    """
+    template = get_template(overlay_name)
+    rendered_template_file = os.path.join(
+        target_dir,
+        os.path.basename(overlay_name))
+    with open(rendered_template_file, "w") as fh:
+        fh.write(
+            template.render(get_template_context_from_env()))
+    return rendered_template_file
+
+
+def render_overlays(bundle, target_dir):
+    """Render the overlays for the given bundle in the directory provided
+
+    :param bundle: Name of bundle being deployed
+    :type bundle: str
+    :param target_dir: Directory to render overlay in
+    :type overlay_name: str
+    :returns: Path to rendered overlay
+    :rtype: str
+    """
+    overlays = []
+    for overlay in DEFAULT_OVERLAYS + [bundle]:
+        rendered_overlay = render_overlay(overlay, target_dir)
+        if rendered_overlay:
+            overlays.append(rendered_overlay)
+    return overlays
+
 
 def deploy_bundle(bundle, model):
     """Deploy the given bundle file in the specified model
@@ -17,7 +143,11 @@ def deploy_bundle(bundle, model):
     :type model: str
     """
     logging.info("Deploying bundle {}".format(bundle))
-    subprocess.check_call(['juju', 'deploy', '-m', model, bundle])
+    cmd = ['juju', 'deploy', '-m', model, bundle]
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        for overlay in render_overlays(bundle, tmpdirname):
+            cmd.extend(['--overlay', overlay])
+        subprocess.check_call(cmd)
 
 
 def deploy(bundle, model, wait=True):