Merge branch 'master' into hacluster-scaleback

requirements.txt

@@ -29,6 +29,7 @@ python-ceilometerclient
 python-cinderclient
 python-glanceclient
 python-heatclient
+python-ironicclient
 python-keystoneclient
 python-manilaclient
 python-neutronclient

setup.py

@@ -44,6 +44,7 @@ install_require = [
     'python-barbicanclient>=4.0.1,<5.0.0',
     'python-designateclient>=1.5,<3.0.0',
     'python-heatclient<2.0.0',
+    'python-ironicclient',
     'python-glanceclient<3.0.0',
     'python-keystoneclient<3.22.0',
     'python-manilaclient<2.0.0',

unit_tests/charm_tests/test_tempest.py

@@ -0,0 +1,66 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import unittest
+
+import zaza.openstack.charm_tests.tempest.setup as tempest_setup
+
+
+class TestTempestSetup(unittest.TestCase):
+    """Test class to encapsulate testing Mysql test utils."""
+
+    def setUp(self):
+        super(TestTempestSetup, self).setUp()
+
+    def test_add_environment_var_config_with_missing_variable(self):
+        ctxt = {}
+        with self.assertRaises(Exception) as context:
+            tempest_setup.add_environment_var_config(ctxt, ['swift'])
+        self.assertEqual(
+            ('Environment variables [TEST_SWIFT_IP] must all be '
+             'set to run this test'),
+            str(context.exception))
+
+    @mock.patch.object(tempest_setup.deployment_env, 'get_deployment_context')
+    def test_add_environment_var_config_with_all_variables(
+            self,
+            get_deployment_context):
+        ctxt = {}
+        get_deployment_context.return_value = {
+            'TEST_GATEWAY': 'test',
+            'TEST_CIDR_EXT': 'test',
+            'TEST_FIP_RANGE': 'test',
+            'TEST_NAME_SERVER': 'test',
+            'TEST_CIDR_PRIV': 'test',
+        }
+        tempest_setup.add_environment_var_config(ctxt, ['neutron'])
+        self.assertEqual(ctxt['test_gateway'], 'test')
+
+    @mock.patch.object(tempest_setup.deployment_env, 'get_deployment_context')
+    def test_add_environment_var_config_with_some_variables(
+            self,
+            get_deployment_context):
+        ctxt = {}
+        get_deployment_context.return_value = {
+            'TEST_GATEWAY': 'test',
+            'TEST_NAME_SERVER': 'test',
+            'TEST_CIDR_PRIV': 'test',
+        }
+        with self.assertRaises(Exception) as context:
+            tempest_setup.add_environment_var_config(ctxt, ['neutron'])
+        self.assertEqual(
+            ('Environment variables [TEST_CIDR_EXT, TEST_FIP_RANGE] must '
+             'all be set to run this test'),
+            str(context.exception))

unit_tests/charm_tests/test_utils.py

@@ -12,21 +12,190 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
+from unittest import mock
+
 import zaza.openstack.charm_tests.test_utils as test_utils
 
-from unittest.mock import patch
+import unit_tests.utils as ut_utils
 
 
-class TestOpenStackBaseTest(unittest.TestCase):
+class TestBaseCharmTest(ut_utils.BaseTestCase):
 
-    @patch.object(test_utils.openstack_utils, 'get_cacert')
-    @patch.object(test_utils.openstack_utils, 'get_overcloud_keystone_session')
-    @patch.object(test_utils.BaseCharmTest, 'setUpClass')
-    def test_setUpClass(self, _setUpClass, _get_ovcks, _get_cacert):
+    def setUp(self):
+        super(TestBaseCharmTest, self).setUp()
+        self.target = test_utils.BaseCharmTest()
+
+    def patch_target(self, attr, return_value=None):
+        mocked = mock.patch.object(self.target, attr)
+        self._patches[attr] = mocked
+        started = mocked.start()
+        started.return_value = return_value
+        self._patches_start[attr] = started
+        setattr(self, attr, started)
+
+    def test_get_my_tests_options(self):
+
+        class FakeTest(test_utils.BaseCharmTest):
+
+            def method(self, test_config):
+                self.test_config = test_config
+                return self.get_my_tests_options('aKey', 'aDefault')
+
+        f = FakeTest()
+        self.assertEquals(f.method({}), 'aDefault')
+        self.assertEquals(f.method({
+            'tests_options': {
+                'unit_tests.charm_tests.test_utils.'
+                'FakeTest.method.aKey': 'aValue',
+            },
+        }), 'aValue')
+
+    def test_config_change(self):
+        default_config = {'fakeKey': 'testProvidedDefault'}
+        alterna_config = {'fakeKey': 'testProvidedAlterna'}
+        self.target.model_name = 'aModel'
+        self.target.test_config = {}
+        self.patch_target('config_current')
+        self.config_current.return_value = default_config
+        self.patch_object(test_utils.model, 'set_application_config')
+        self.patch_object(test_utils.model, 'wait_for_agent_status')
+        self.patch_object(test_utils.model, 'wait_for_application_states')
+        self.patch_object(test_utils.model, 'block_until_all_units_idle')
+        with self.target.config_change(
+                default_config, alterna_config, application_name='anApp'):
+            self.set_application_config.assert_called_once_with(
+                'anApp', alterna_config, model_name='aModel')
+            self.wait_for_agent_status.assert_called_once_with(
+                model_name='aModel')
+            self.wait_for_application_states.assert_called_once_with(
+                model_name='aModel', states={})
+            self.block_until_all_units_idle.assert_called_once_with()
+        # after yield we will have different calls than the above, measure both
+        self.set_application_config.assert_has_calls([
+            mock.call('anApp', alterna_config, model_name='aModel'),
+            mock.call('anApp', default_config, model_name='aModel'),
+        ])
+        self.wait_for_application_states.assert_has_calls([
+            mock.call(model_name='aModel', states={}),
+            mock.call(model_name='aModel', states={}),
+        ])
+        self.block_until_all_units_idle.assert_has_calls([
+            mock.call(),
+            mock.call(),
+        ])
+        # confirm operation with `reset_to_charm_default`
+        self.set_application_config.reset_mock()
+        self.wait_for_agent_status.reset_mock()
+        self.wait_for_application_states.reset_mock()
+        self.patch_object(test_utils.model, 'reset_application_config')
+        with self.target.config_change(
+                default_config, alterna_config, application_name='anApp',
+                reset_to_charm_default=True):
+            self.set_application_config.assert_called_once_with(
+                'anApp', alterna_config, model_name='aModel')
+            # we want to assert this not to be called after yield
+            self.set_application_config.reset_mock()
+        self.assertFalse(self.set_application_config.called)
+        self.reset_application_config.assert_called_once_with(
+            'anApp', list(alterna_config.keys()), model_name='aModel')
+        self.wait_for_application_states.assert_has_calls([
+            mock.call(model_name='aModel', states={}),
+            mock.call(model_name='aModel', states={}),
+        ])
+        self.block_until_all_units_idle.assert_has_calls([
+            mock.call(),
+            mock.call(),
+        ])
+        # confirm operation where both default and alternate config passed in
+        # are the same. This is used to set config and not change it back.
+        self.set_application_config.reset_mock()
+        self.wait_for_agent_status.reset_mock()
+        self.wait_for_application_states.reset_mock()
+        self.reset_application_config.reset_mock()
+        with self.target.config_change(
+                alterna_config, alterna_config, application_name='anApp'):
+            self.set_application_config.assert_called_once_with(
+                'anApp', alterna_config, model_name='aModel')
+            # we want to assert these not to be called after yield
+            self.set_application_config.reset_mock()
+            self.wait_for_agent_status.reset_mock()
+            self.wait_for_application_states.reset_mock()
+        self.assertFalse(self.set_application_config.called)
+        self.assertFalse(self.reset_application_config.called)
+        self.assertFalse(self.wait_for_agent_status.called)
+        self.assertFalse(self.wait_for_application_states.called)
+
+    def test_separate_non_string_config(self):
+        intended_cfg_keys = ['foo2', 'foo3', 'foo4', 'foo5']
+        current_config_mock = {
+            'foo2': None,
+            'foo3': 'old_bar3',
+            'foo4': None,
+            'foo5': 'old_bar5',
+        }
+        self.patch_target('config_current')
+        self.config_current.return_value = current_config_mock
+        non_string_type_keys = ['foo2', 'foo3', 'foo4']
+        expected_result_filtered = {
+            'foo3': 'old_bar3',
+            'foo5': 'old_bar5',
+        }
+        expected_result_special = {
+            'foo2': None,
+            'foo4': None,
+        }
+        current, non_string = (
+            self.target.config_current_separate_non_string_type_keys(
+                non_string_type_keys, intended_cfg_keys, 'application_name')
+        )
+
+        self.assertEqual(expected_result_filtered, current)
+        self.assertEqual(expected_result_special, non_string)
+
+        self.config_current.assert_called_once_with(
+            'application_name', intended_cfg_keys)
+
+    def test_separate_special_config_None_params(self):
+        current_config_mock = {
+            'foo1': 'old_bar1',
+            'foo2': None,
+            'foo3': 'old_bar3',
+            'foo4': None,
+            'foo5': 'old_bar5',
+        }
+        self.patch_target('config_current')
+        self.config_current.return_value = current_config_mock
+        non_string_type_keys = ['foo2', 'foo3', 'foo4']
+        expected_result_filtered = {
+            'foo1': 'old_bar1',
+            'foo3': 'old_bar3',
+            'foo5': 'old_bar5',
+        }
+        expected_result_special = {
+            'foo2': None,
+            'foo4': None,
+        }
+        current, non_string = (
+            self.target.config_current_separate_non_string_type_keys(
+                non_string_type_keys)
+        )
+
+        self.assertEqual(expected_result_filtered, current)
+        self.assertEqual(expected_result_special, non_string)
+
+        self.config_current.assert_called_once_with(None, None)
+
+
+class TestOpenStackBaseTest(ut_utils.BaseTestCase):
+
+    def test_setUpClass(self):
+        self.patch_object(test_utils.openstack_utils, 'get_cacert')
+        self.patch_object(test_utils.openstack_utils,
+                          'get_overcloud_keystone_session')
+        self.patch_object(test_utils.BaseCharmTest, 'setUpClass')
 
         class MyTestClass(test_utils.OpenStackBaseTest):
             model_name = 'deadbeef'
 
         MyTestClass.setUpClass('foo', 'bar')
-        _setUpClass.assert_called_with('foo', 'bar')
+        self.setUpClass.assert_called_with('foo', 'bar')

unit_tests/utilities/test_zaza_utilities_ceph.py

@@ -116,3 +116,20 @@ class TestCephUtils(ut_utils.BaseTestCase):
         with self.assertRaises(model.CommandRunFailed):
             ceph_utils.get_rbd_hash('aunit', 'apool', 'aimage',
                                     model_name='amodel')
+
+    def test_pools_from_broker_req(self):
+        self.patch_object(ceph_utils.zaza_juju, 'get_relation_from_unit')
+        self.get_relation_from_unit.return_value = {
+            'broker_req': (
+                '{"api-version": 1, "ops": ['
+                '{"op": "create-pool", "name": "cinder-ceph", '
+                '"compression-mode": null},'
+                '{"op": "create-pool", "name": "cinder-ceph", '
+                '"compression-mode": "aggressive"}]}'),
+        }
+        self.assertEquals(
+            ceph_utils.get_pools_from_broker_req(
+                'anApplication', 'aModelName'),
+            ['cinder-ceph'])
+        self.get_relation_from_unit.assert_called_once_with(
+            'ceph-mon', 'anApplication', None, model_name='aModelName')

unit_tests/utilities/test_zaza_utilities_openstack.py

@@ -289,6 +289,15 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         openstack_utils.get_undercloud_keystone_session()
         self.get_keystone_session.assert_called_once_with(_auth, verify=None)
 
+    def test_get_nova_session_client(self):
+        session_mock = mock.MagicMock()
+        self.patch_object(openstack_utils.novaclient_client, "Client")
+        openstack_utils.get_nova_session_client(session_mock)
+        self.Client.assert_called_once_with(2, session=session_mock)
+        self.Client.reset_mock()
+        openstack_utils.get_nova_session_client(session_mock, version=2.56)
+        self.Client.assert_called_once_with(2.56, session=session_mock)
+
     def test_get_urllib_opener(self):
         self.patch_object(openstack_utils.urllib.request, "ProxyHandler")
         self.patch_object(openstack_utils.urllib.request, "HTTPHandler")
@@ -369,12 +378,15 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
                 'e01df65a')
 
     def test__resource_reaches_status_bespoke(self):
+        client_mock = mock.MagicMock()
         resource_mock = mock.MagicMock()
-        resource_mock.get.return_value = mock.MagicMock(status='readyish')
+        resource_mock.special_status = 'readyish'
+        client_mock.get.return_value = resource_mock
         openstack_utils._resource_reaches_status(
-            resource_mock,
+            client_mock,
             'e01df65a',
-            'readyish')
+            'readyish',
+            resource_attribute='special_status')
 
     def test__resource_reaches_status_bespoke_fail(self):
         resource_mock = mock.MagicMock()
@@ -504,7 +516,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
             glance_mock.images.upload.assert_called_once_with(
                 '9d1125af',
                 f(),
-            )
+                backend=None)
             self.resource_reaches_status.assert_called_once_with(
                 glance_mock.images,
                 '9d1125af',
@@ -529,7 +541,11 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.upload_image_to_glance.assert_called_once_with(
             glance_mock,
             'wibbly/c.img',
-            'bob')
+            'bob',
+            backend=None,
+            disk_format='qcow2',
+            visibility='public',
+            container_format='bare')
 
     def test_create_image_pass_directory(self):
         glance_mock = mock.MagicMock()
@@ -549,7 +565,11 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.upload_image_to_glance.assert_called_once_with(
             glance_mock,
             'tests/c.img',
-            'bob')
+            'bob',
+            backend=None,
+            disk_format='qcow2',
+            visibility='public',
+            container_format='bare')
         self.gettempdir.assert_not_called()
 
     def test_create_ssh_key(self):
@@ -1227,6 +1247,13 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.get_application.side_effect = [KeyError, KeyError]
         self.assertFalse(openstack_utils.ovn_present())
 
+    def test_ngw_present(self):
+        self.patch_object(openstack_utils.model, 'get_application')
+        self.get_application.side_effect = None
+        self.assertTrue(openstack_utils.ngw_present())
+        self.get_application.side_effect = KeyError
+        self.assertFalse(openstack_utils.ngw_present())
+
     def test_configure_gateway_ext_port(self):
         # FIXME: this is not a complete unit test for the function as one did
         # not exist at all I'm adding this to test one bit and we'll add more
@@ -1234,10 +1261,12 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.patch_object(openstack_utils, 'deprecated_external_networking')
         self.patch_object(openstack_utils, 'dvr_enabled')
         self.patch_object(openstack_utils, 'ovn_present')
+        self.patch_object(openstack_utils, 'ngw_present')
         self.patch_object(openstack_utils, 'get_gateway_uuids')
         self.patch_object(openstack_utils, 'get_admin_net')
-        self.dvr_enabled = False
-        self.ovn_present = False
+        self.dvr_enabled.return_value = False
+        self.ovn_present.return_value = False
+        self.ngw_present.return_value = True
         self.get_admin_net.return_value = {'id': 'fakeid'}
 
         novaclient = mock.MagicMock()

zaza/openstack/charm_tests/ceph/iscsi/setup.py

@@ -19,12 +19,12 @@ import zaza.model
 
 def basic_guest_setup():
     """Run basic setup for iscsi guest."""
-    unit = zaza.model.get_units('ubuntu')[0]
-    setup_cmds = [
-        "apt install --yes open-iscsi multipath-tools",
-        "systemctl start iscsi",
-        "systemctl start iscsid"]
-    for cmd in setup_cmds:
-        zaza.model.run_on_unit(
-            unit.entity_id,
-            cmd)
+    for unit in zaza.model.get_units('ubuntu'):
+        setup_cmds = [
+            "apt install --yes open-iscsi multipath-tools",
+            "systemctl start iscsi",
+            "systemctl start iscsid"]
+        for cmd in setup_cmds:
+            zaza.model.run_on_unit(
+                unit.entity_id,
+                cmd)

zaza/openstack/charm_tests/ceph/iscsi/tests.py

@@ -26,10 +26,19 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
     """Class for `ceph-iscsi` tests."""
 
     GW_IQN = "iqn.2003-03.com.canonical.iscsi-gw:iscsi-igw"
-    DATA_POOL_NAME = 'superssd'
+    DATA_POOL_NAME = 'zaza_rep_pool'
+    EC_PROFILE_NAME = 'zaza_iscsi'
+    EC_DATA_POOL = 'zaza_ec_data_pool'
+    EC_METADATA_POOL = 'zaza_ec_metadata_pool'
 
     def get_client_initiatorname(self, unit):
-        """Return the initiatorname for the given unit."""
+        """Return the initiatorname for the given unit.
+
+        :param unit_name: Name of unit to match
+        :type unit: str
+        :returns: Initiator name
+        :rtype: str
+        """
         generic_utils.assertRemoteRunOK(zaza.model.run_on_unit(
             unit,
             ('cp /etc/iscsi/initiatorname.iscsi /tmp; '
@@ -48,18 +57,20 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
                 initiatorname = line.split('=')[1].rstrip()
         return initiatorname
 
-    def get_ctxt(self):
-        """Generate a context for running gwcli commands to create a target."""
+    def get_base_ctxt(self):
+        """Generate a context for running gwcli commands to create a target.
+
+        :returns: Base gateway context
+        :rtype: Dict
+        """
         gw_units = zaza.model.get_units('ceph-iscsi')
-        client_units = zaza.model.get_units('ubuntu')
-        client = client_units[0]
-        self.get_client_initiatorname(client.entity_id)
         primary_gw = gw_units[0]
         secondary_gw = gw_units[1]
         host_names = generic_utils.get_unit_hostnames(gw_units, fqdn=True)
+        client_entity_ids = [
+            u.entity_id for u in zaza.model.get_units('ubuntu')]
         ctxt = {
-            'pool_name': self.DATA_POOL_NAME,
-            'client_entity_id': client.entity_id,
+            'client_entity_ids': sorted(client_entity_ids),
             'gw_iqn': self.GW_IQN,
             'gw1_ip': primary_gw.public_address,
             'gw1_hostname': host_names[primary_gw.entity_id],
@@ -67,13 +78,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
             'gw2_ip': secondary_gw.public_address,
             'gw2_hostname': host_names[secondary_gw.entity_id],
             'gw2_entity_id': secondary_gw.entity_id,
-            'img_size': '1G',
-            'img_name': 'disk_1',
-            'chap_username': 'myiscsiusername',
-            'chap_password': 'myiscsipassword',
             'chap_creds': 'username={chap_username} password={chap_password}',
-            'client_initiatorname': self.get_client_initiatorname(
-                client.entity_id),
             'gwcli_gw_dir': '/iscsi-targets/{gw_iqn}/gateways',
             'gwcli_hosts_dir': '/iscsi-targets/{gw_iqn}/hosts',
             'gwcli_disk_dir': '/disks',
@@ -84,8 +89,16 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
     def run_commands(self, unit_name, commands, ctxt):
         """Run commands on unit.
 
-        Apply context to commands until all variables have been replaced, then
-        run the command on the given unit.
+        Iterate over each command and apply the context to the command, then
+        run the command on the supplied unit.
+
+        :param unit_name: Name of unit to match
+        :type unit: str
+        :param commands: List of commands to run.
+        :type commands: List[str]
+        :param ctxt: Context to apply to each command.
+        :type ctxt: Dict
+        :raises: AssertionError
         """
         for _cmd in commands:
             cmd = _cmd.format(**ctxt)
@@ -94,7 +107,11 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
                 cmd))
 
     def create_iscsi_target(self, ctxt):
-        """Create target on gateway."""
+        """Create target on gateway.
+
+        :param ctxt: Base gateway context
+        :type ctxt: Dict
+        """
         generic_utils.assertActionRanOK(zaza.model.run_action_on_leader(
             'ceph-iscsi',
             'create-target',
@@ -103,7 +120,8 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
                     ctxt['gw1_entity_id'],
                     ctxt['gw2_entity_id']),
                 'iqn': self.GW_IQN,
-                'pool-name': self.DATA_POOL_NAME,
+                'rbd-pool-name': ctxt.get('pool_name', ''),
+                'ec-rbd-metadata-pool': ctxt.get('ec_meta_pool_name', ''),
                 'image-size': ctxt['img_size'],
                 'image-name': ctxt['img_name'],
                 'client-initiatorname': ctxt['client_initiatorname'],
@@ -111,8 +129,13 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
                 'client-password': ctxt['chap_password']
             }))
 
-    def mount_iscsi_target(self, ctxt):
-        """Mount iscsi target on client."""
+    def login_iscsi_target(self, ctxt):
+        """Login to the iscsi target on client.
+
+        :param ctxt: Base gateway context
+        :type ctxt: Dict
+        """
+        logging.info("Logging in to iscsi target")
         base_op_cmd = ('iscsiadm --mode node --targetname {gw_iqn} '
                        '--op=update ').format(**ctxt)
         setup_cmds = [
@@ -123,14 +146,67 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
             'iscsiadm --mode node --targetname {gw_iqn} --login']
         self.run_commands(ctxt['client_entity_id'], setup_cmds, ctxt)
 
-    def check_client_device(self, ctxt):
-        """Wait for multipath device to appear on client."""
+    def logout_iscsi_targets(self, ctxt):
+        """Logout of iscsi target on client.
+
+        :param ctxt: Base gateway context
+        :type ctxt: Dict
+        """
+        logging.info("Logging out of iscsi target")
+        logout_cmds = [
+            'iscsiadm --mode node --logoutall=all']
+        self.run_commands(ctxt['client_entity_id'], logout_cmds, ctxt)
+
+    def check_client_device(self, ctxt, init_client=True):
+        """Wait for multipath device to appear on client and test access.
+
+        :param ctxt: Base gateway context
+        :type ctxt: Dict
+        :param init_client: Initialise client if this is the first time it has
+                            been used.
+        :type init_client: bool
+        """
+        logging.info("Checking multipath device is present.")
+        device_ctxt = {
+            'bdevice': '/dev/dm-0',
+            'mount_point': '/mnt/iscsi',
+            'test_file': '/mnt/iscsi/test.data'}
+        ls_bdevice_cmd = 'ls -l {bdevice}'
+        mkfs_cmd = 'mke2fs {bdevice}'
+        mkdir_cmd = 'mkdir {mount_point}'
+        mount_cmd = 'mount {bdevice} {mount_point}'
+        umount_cmd = 'umount {mount_point}'
+        check_mounted_cmd = 'mountpoint {mount_point}'
+        write_cmd = 'truncate -s 1M {test_file}'
+        check_file = 'ls -l {test_file}'
+        if init_client:
+            commands = [
+                mkfs_cmd,
+                mkdir_cmd,
+                mount_cmd,
+                check_mounted_cmd,
+                write_cmd,
+                check_file,
+                umount_cmd]
+        else:
+            commands = [
+                mount_cmd,
+                check_mounted_cmd,
+                check_file,
+                umount_cmd]
+
         async def check_device_present():
             run = await zaza.model.async_run_on_unit(
                 ctxt['client_entity_id'],
-                'ls -l /dev/dm-0')
-            return '/dev/dm-0' in run['Stdout']
+                ls_bdevice_cmd.format(bdevice=device_ctxt['bdevice']))
+            return device_ctxt['bdevice'] in run['stdout']
+
+        logging.info("Checking {} is present on {}".format(
+            device_ctxt['bdevice'],
+            ctxt['client_entity_id']))
         zaza.model.block_until(check_device_present)
+        logging.info("Checking mounting device and access")
+        self.run_commands(ctxt['client_entity_id'], commands, device_ctxt)
 
     def create_data_pool(self):
         """Create data pool to back iscsi targets."""
@@ -140,13 +216,94 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
             action_params={
                 'name': self.DATA_POOL_NAME}))
 
+    def create_ec_data_pool(self):
+        """Create data pool to back iscsi targets."""
+        generic_utils.assertActionRanOK(zaza.model.run_action_on_leader(
+            'ceph-mon',
+            'create-erasure-profile',
+            action_params={
+                'name': self.EC_PROFILE_NAME,
+                'coding-chunks': 2,
+                'data-chunks': 4,
+                'plugin': 'jerasure'}))
+        generic_utils.assertActionRanOK(zaza.model.run_action_on_leader(
+            'ceph-mon',
+            'create-pool',
+            action_params={
+                'name': self.EC_DATA_POOL,
+                'pool-type': 'erasure-coded',
+                'allow-ec-overwrites': True,
+                'erasure-profile-name': self.EC_PROFILE_NAME}))
+        generic_utils.assertActionRanOK(zaza.model.run_action_on_leader(
+            'ceph-mon',
+            'create-pool',
+            action_params={
+                'name': self.EC_METADATA_POOL}))
+
+    def run_client_checks(self, test_ctxt):
+        """Check access to mulipath device.
+
+        Write a filesystem to device, mount it and write data. Then unmount
+        and logout the iscsi target, finally reconnect and remount checking
+        data is still present.
+
+        :param test_ctxt: Test context.
+        :type test_ctxt: Dict
+        """
+        self.create_iscsi_target(test_ctxt)
+        self.login_iscsi_target(test_ctxt)
+        self.check_client_device(test_ctxt, init_client=True)
+        self.logout_iscsi_targets(test_ctxt)
+        self.login_iscsi_target(test_ctxt)
+        self.check_client_device(test_ctxt, init_client=False)
+
     def test_create_and_mount_volume(self):
         """Test creating a target and mounting it on a client."""
         self.create_data_pool()
-        ctxt = self.get_ctxt()
-        self.create_iscsi_target(ctxt)
-        self.mount_iscsi_target(ctxt)
-        self.check_client_device(ctxt)
+        ctxt = self.get_base_ctxt()
+        client_entity_id = ctxt['client_entity_ids'][0]
+        ctxt.update({
+            'client_entity_id': client_entity_id,
+            'client_initiatorname': self.get_client_initiatorname(
+                client_entity_id),
+            'pool_name': self.DATA_POOL_NAME,
+            'chap_username': 'myiscsiusername1',
+            'chap_password': 'myiscsipassword1',
+            'img_size': '1G',
+            'img_name': 'disk_rep_1'})
+        self.run_client_checks(ctxt)
+
+    def test_create_and_mount_ec_backed_volume(self):
+        """Test creating an EC backed target and mounting it on a client."""
+        self.create_ec_data_pool()
+        ctxt = self.get_base_ctxt()
+        client_entity_id = ctxt['client_entity_ids'][1]
+        ctxt.update({
+            'client_entity_id': client_entity_id,
+            'client_initiatorname': self.get_client_initiatorname(
+                client_entity_id),
+            'pool_name': self.EC_DATA_POOL,
+            'ec_meta_pool_name': self.EC_METADATA_POOL,
+            'chap_username': 'myiscsiusername2',
+            'chap_password': 'myiscsipassword2',
+            'img_size': '2G',
+            'img_name': 'disk_ec_1'})
+        self.run_client_checks(ctxt)
+
+    def test_create_and_mount_volume_default_pool(self):
+        """Test creating a target and mounting it on a client."""
+        self.create_data_pool()
+        ctxt = self.get_base_ctxt()
+        client_entity_id = ctxt['client_entity_ids'][2]
+        ctxt.update({
+            'client_entity_id': client_entity_id,
+            'client_initiatorname': self.get_client_initiatorname(
+                client_entity_id),
+            'chap_username': 'myiscsiusername3',
+            'chap_password': 'myiscsipassword3',
+            'img_size': '3G',
+            'img_name': 'disk_default_1'})
+        self.run_client_checks(ctxt)
 
     def test_pause_resume(self):
         """Test pausing and resuming a unit."""

zaza/openstack/charm_tests/ceph/osd/tests.py

@@ -16,6 +16,9 @@
 
 import logging
 import unittest
+import re
+
+from copy import deepcopy
 
 import zaza.openstack.charm_tests.test_utils as test_utils
 import zaza.model as zaza_model
@@ -47,3 +50,226 @@ class SecurityTest(unittest.TestCase):
             expected_passes,
             expected_failures,
             expected_to_pass=True)
+
+
+class OsdService:
+    """Simple representation of ceph-osd systemd service."""
+
+    def __init__(self, id_):
+        """
+        Init service using its ID.
+
+        e.g.: id_=1 -> ceph-osd@1
+        """
+        self.id = id_
+        self.name = 'ceph-osd@{}'.format(id_)
+
+
+async def async_wait_for_service_status(unit_name, services, target_status,
+                                        model_name=None, timeout=2700):
+    """Wait for all services on the unit to be in the desired state.
+
+    Note: This function emulates the
+    `zaza.model.async_block_until_service_status` function, but it's using
+    `systemctl is-active` command instead of `pidof/pgrep` of the original
+    function.
+
+    :param unit_name: Name of unit to run action on
+    :type unit_name: str
+    :param services: List of services to check
+    :type services: List[str]
+    :param target_status: State services must be in (stopped or running)
+    :type target_status: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :param timeout: Time to wait for status to be achieved
+    :type timeout: int
+    """
+    async def _check_service():
+        services_ok = True
+        for service in services:
+            command = r"systemctl is-active '{}'".format(service)
+            out = await zaza_model.async_run_on_unit(
+                unit_name,
+                command,
+                model_name=model_name,
+                timeout=timeout)
+            response = out['Stdout'].strip()
+
+            if target_status == "running" and response == 'active':
+                continue
+            elif target_status == "stopped" and response == 'inactive':
+                continue
+            else:
+                services_ok = False
+                break
+
+        return services_ok
+
+    accepted_states = ('stopped', 'running')
+    if target_status not in accepted_states:
+        raise RuntimeError('Invalid target state "{}". Accepted states: '
+                           '{}'.format(target_status, accepted_states))
+
+    async with zaza_model.run_in_model(model_name):
+        await zaza_model.async_block_until(_check_service, timeout=timeout)
+
+
+wait_for_service = zaza_model.sync_wrapper(async_wait_for_service_status)
+
+
+class ServiceTest(unittest.TestCase):
+    """ceph-osd systemd service tests."""
+
+    TESTED_UNIT = 'ceph-osd/0'  # This can be any ceph-osd unit in the model
+    SERVICE_PATTERN = re.compile(r'ceph-osd@(?P<service_id>\d+)\.service')
+
+    def __init__(self, methodName='runTest'):
+        """Initialize Test Case."""
+        super(ServiceTest, self).__init__(methodName)
+        self._available_services = None
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running ceph service tests."""
+        super(ServiceTest, cls).setUpClass()
+
+    def setUp(self):
+        """Run test setup."""
+        # Note: This counter reset is needed because ceph-osd service is
+        #       limited to 3 restarts per 30 mins which is insufficient
+        #       when running functional tests for 'service' action. This
+        #       limitation is defined in /lib/systemd/system/ceph-osd@.service
+        #       in section [Service] with options 'StartLimitInterval' and
+        #       'StartLimitBurst'
+        reset_counter = 'systemctl reset-failed'
+        zaza_model.run_on_unit(self.TESTED_UNIT, reset_counter)
+
+    def tearDown(self):
+        """Start ceph-osd services after each test.
+
+        This ensures that the environment is ready for the next tests.
+        """
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service',
+                                       action_params={'start': 'all'},
+                                       raise_on_failure=True)
+
+    @property
+    def available_services(self):
+        """Return list of all ceph-osd services present on the TESTED_UNIT."""
+        if self._available_services is None:
+            self._available_services = self._fetch_osd_services()
+        return self._available_services
+
+    def _fetch_osd_services(self):
+        """Fetch all ceph-osd services present on the TESTED_UNIT."""
+        service_list = []
+        service_list_cmd = 'systemctl list-units --full --all ' \
+                           '--no-pager -t service'
+        result = zaza_model.run_on_unit(self.TESTED_UNIT, service_list_cmd)
+        for line in result['Stdout'].split('\n'):
+            service_name = self.SERVICE_PATTERN.search(line)
+            if service_name:
+                service_id = int(service_name.group('service_id'))
+                service_list.append(OsdService(service_id))
+        return service_list
+
+    def test_start_stop_all_by_keyword(self):
+        """Start and Stop all ceph-osd services using keyword 'all'."""
+        service_list = [service.name for service in self.available_services]
+
+        logging.info("Running 'service stop=all' action on {} "
+                     "unit".format(self.TESTED_UNIT))
+        zaza_model.run_action_on_units([self.TESTED_UNIT], 'service',
+                                       action_params={'stop': 'all'})
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_list,
+                         target_status='stopped')
+
+        logging.info("Running 'service start=all' action on {} "
+                     "unit".format(self.TESTED_UNIT))
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service',
+                                       action_params={'start': 'all'})
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_list,
+                         target_status='running')
+
+    def test_start_stop_all_by_list(self):
+        """Start and Stop all ceph-osd services using explicit list."""
+        service_list = [service.name for service in self.available_services]
+        service_ids = [str(service.id) for service in self.available_services]
+        action_params = ','.join(service_ids)
+
+        logging.info("Running 'service stop={}' action on {} "
+                     "unit".format(action_params, self.TESTED_UNIT))
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service',
+                                       action_params={'stop': action_params})
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_list,
+                         target_status='stopped')
+
+        logging.info("Running 'service start={}' action on {} "
+                     "unit".format(action_params, self.TESTED_UNIT))
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service',
+                                       action_params={'start': action_params})
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_list,
+                         target_status='running')
+
+    def test_stop_specific(self):
+        """Stop only specified ceph-osd service."""
+        if len(self.available_services) < 2:
+            raise unittest.SkipTest('This test can be performed only if '
+                                    'there\'s more than one ceph-osd service '
+                                    'present on the tested unit')
+
+        should_run = deepcopy(self.available_services)
+        to_stop = should_run.pop()
+        should_run = [service.name for service in should_run]
+
+        logging.info("Running 'service stop={} on {} "
+                     "unit".format(to_stop.id, self.TESTED_UNIT))
+
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service',
+                                       action_params={'stop': to_stop.id})
+
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=[to_stop.name, ],
+                         target_status='stopped')
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=should_run,
+                         target_status='running')
+
+    def test_start_specific(self):
+        """Start only specified ceph-osd service."""
+        if len(self.available_services) < 2:
+            raise unittest.SkipTest('This test can be performed only if '
+                                    'there\'s more than one ceph-osd service '
+                                    'present on the tested unit')
+
+        service_names = [service.name for service in self.available_services]
+        should_stop = deepcopy(self.available_services)
+        to_start = should_stop.pop()
+        should_stop = [service.name for service in should_stop]
+
+        logging.info("Stopping all running ceph-osd services")
+        service_stop_cmd = 'systemctl stop ceph-osd.target'
+        zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd)
+
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_names,
+                         target_status='stopped')
+
+        logging.info("Running 'service start={} on {} "
+                     "unit".format(to_start.id, self.TESTED_UNIT))
+
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service',
+                                       action_params={'start': to_start.id})
+
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=[to_start.name, ],
+                         target_status='running')
+
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=should_stop,
+                         target_status='stopped')

zaza/openstack/charm_tests/ceph/tests.py

@@ -872,3 +872,141 @@ def _get_mon_count_from_prometheus(prometheus_ip):
     response = client.get(url)
     logging.debug("Prometheus response: {}".format(response.json()))
     return response.json()['data']['result'][0]['value'][1]
+
+
+class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest):
+    """Test charm handling of bluestore compression configuration options."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Perform class one time initialization."""
+        super(BlueStoreCompressionCharmOperation, cls).setUpClass()
+        cls.current_release = zaza_openstack.get_os_release(
+            zaza_openstack.get_current_os_release_pair(
+                application='ceph-mon'))
+        cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky')
+
+    def setUp(self):
+        """Perform common per test initialization steps."""
+        super(BlueStoreCompressionCharmOperation, self).setUp()
+
+        # determine if the tests should be run or not
+        logging.debug('os_release: {} >= {} = {}'
+                      .format(self.current_release,
+                              self.bionic_rocky,
+                              self.current_release >= self.bionic_rocky))
+        self.mimic_or_newer = self.current_release >= self.bionic_rocky
+
+    def _assert_pools_properties(self, pools, pools_detail,
+                                 expected_properties, log_func=logging.info):
+        """Check properties on a set of pools.
+
+        :param pools: List of pool names to check.
+        :type pools: List[str]
+        :param pools_detail: List of dictionaries with pool detail
+        :type pools_detail List[Dict[str,any]]
+        :param expected_properties: Properties to check and their expected
+                                    values.
+        :type expected_properties: Dict[str,any]
+        :returns: Nothing
+        :raises: AssertionError
+        """
+        for pool in pools:
+            for pd in pools_detail:
+                if pd['pool_name'] == pool:
+                    if 'options' in expected_properties:
+                        for k, v in expected_properties['options'].items():
+                            self.assertEquals(pd['options'][k], v)
+                            log_func("['options']['{}'] == {}".format(k, v))
+                    for k, v in expected_properties.items():
+                        if k == 'options':
+                            continue
+                        self.assertEquals(pd[k], v)
+                        log_func("{} == {}".format(k, v))
+
+    def test_configure_compression(self):
+        """Enable compression and validate properties flush through to pool."""
+        if not self.mimic_or_newer:
+            logging.info('Skipping test, Mimic or newer required.')
+            return
+        if self.application_name == 'ceph-osd':
+            # The ceph-osd charm itself does not request pools, neither does
+            # the BlueStore Compression configuration options it have affect
+            # pool properties.
+            logging.info('test does not apply to ceph-osd charm.')
+            return
+        elif self.application_name == 'ceph-radosgw':
+            # The Ceph RadosGW creates many light weight pools to keep track of
+            # metadata, we only compress the pool containing actual data.
+            app_pools = ['.rgw.buckets.data']
+        else:
+            # Retrieve which pools the charm under test has requested skipping
+            # metadata pools as they are deliberately not compressed.
+            app_pools = [
+                pool
+                for pool in zaza_ceph.get_pools_from_broker_req(
+                    self.application_name, model_name=self.model_name)
+                if 'metadata' not in pool
+            ]
+
+        ceph_pools_detail = zaza_ceph.get_ceph_pool_details(
+            model_name=self.model_name)
+
+        logging.debug('BEFORE: {}'.format(ceph_pools_detail))
+        try:
+            logging.info('Checking Ceph pool compression_mode prior to change')
+            self._assert_pools_properties(
+                app_pools, ceph_pools_detail,
+                {'options': {'compression_mode': 'none'}})
+        except KeyError:
+            logging.info('property does not exist on pool, which is OK.')
+        logging.info('Changing "bluestore-compression-mode" to "force" on {}'
+                     .format(self.application_name))
+        with self.config_change(
+                {'bluestore-compression-mode': 'none'},
+                {'bluestore-compression-mode': 'force'}):
+            # Retrieve pool details from Ceph after changing configuration
+            ceph_pools_detail = zaza_ceph.get_ceph_pool_details(
+                model_name=self.model_name)
+            logging.debug('CONFIG_CHANGE: {}'.format(ceph_pools_detail))
+            logging.info('Checking Ceph pool compression_mode after to change')
+            self._assert_pools_properties(
+                app_pools, ceph_pools_detail,
+                {'options': {'compression_mode': 'force'}})
+        ceph_pools_detail = zaza_ceph.get_ceph_pool_details(
+            model_name=self.model_name)
+        logging.debug('AFTER: {}'.format(ceph_pools_detail))
+        logging.debug(zaza_juju.get_relation_from_unit(
+            'ceph-mon', self.application_name, None,
+            model_name=self.model_name))
+        logging.info('Checking Ceph pool compression_mode after restoring '
+                     'config to previous value')
+        self._assert_pools_properties(
+            app_pools, ceph_pools_detail,
+            {'options': {'compression_mode': 'none'}})
+
+    def test_invalid_compression_configuration(self):
+        """Set invalid configuration and validate charm response."""
+        if not self.mimic_or_newer:
+            logging.info('Skipping test, Mimic or newer required.')
+            return
+        stored_target_deploy_status = self.test_config.get(
+            'target_deploy_status', {})
+        new_target_deploy_status = stored_target_deploy_status.copy()
+        new_target_deploy_status[self.application_name] = {
+            'workload-status': 'blocked',
+            'workload-status-message': 'Invalid configuration',
+        }
+        if 'target_deploy_status' in self.test_config:
+            self.test_config['target_deploy_status'].update(
+                new_target_deploy_status)
+        else:
+            self.test_config['target_deploy_status'] = new_target_deploy_status
+
+        with self.config_change(
+                {'bluestore-compression-mode': 'none'},
+                {'bluestore-compression-mode': 'PEBCAK'}):
+            logging.info('Charm went into blocked state as expected, restore '
+                         'configuration')
+            self.test_config[
+                'target_deploy_status'] = stored_target_deploy_status

zaza/openstack/charm_tests/cinder_backup/tests.py

@@ -63,7 +63,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
             self.cinder_client.volumes,
             vol_new.id,
             expected_status="available",
-            msg="Volume status wait")
+            msg="Extended volume")
 
     def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self):
         """Create, backup, delete, restore a ceph-backed cinder volume.
@@ -89,29 +89,44 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
 
         self.assertEqual(pool_name, expected_pool)
 
-        # Create ceph-backed cinder volume
-        cinder_vol = self.cinder_client.volumes.create(
-            name='{}-410-vol'.format(self.RESOURCE_PREFIX),
-            size=1)
-        openstack_utils.resource_reaches_status(
-            self.cinder_client.volumes,
-            cinder_vol.id,
-            wait_iteration_max_time=180,
-            stop_after_attempt=30,
-            expected_status='available',
-            msg='Volume status wait')
+        for attempt in tenacity.Retrying(
+                stop=tenacity.stop_after_attempt(3)):
+            with attempt:
+                # Create ceph-backed cinder volume
+                cinder_vol_name = '{}-410-{}-vol'.format(
+                    self.RESOURCE_PREFIX, attempt.retry_state.attempt_number)
+                cinder_vol = self.cinder_client.volumes.create(
+                    name=cinder_vol_name, size=1)
+                openstack_utils.resource_reaches_status(
+                    self.cinder_client.volumes,
+                    cinder_vol.id,
+                    wait_iteration_max_time=180,
+                    stop_after_attempt=15,
+                    expected_status='available',
+                    msg='ceph-backed cinder volume')
+
+                # Back up the volume
+                # NOTE(lourot): sometimes, especially on Mitaka, the backup
+                # remains stuck forever in 'creating' state and the volume in
+                # 'backing-up' state. See lp:1877076
+                # Attempting to create another volume and another backup
+                # usually then succeeds. Release notes and bug trackers show
+                # that many things have been fixed and are still left to be
+                # fixed in this area.
+                # When the backup creation succeeds, it usually does within
+                # 12 minutes.
+                vol_backup_name = '{}-410-{}-backup-vol'.format(
+                    self.RESOURCE_PREFIX, attempt.retry_state.attempt_number)
+                vol_backup = self.cinder_client.backups.create(
+                    cinder_vol.id, name=vol_backup_name)
+                openstack_utils.resource_reaches_status(
+                    self.cinder_client.backups,
+                    vol_backup.id,
+                    wait_iteration_max_time=180,
+                    stop_after_attempt=15,
+                    expected_status='available',
+                    msg='Backup volume')
 
-        # Backup the volume
-        vol_backup = self.cinder_client.backups.create(
-            cinder_vol.id,
-            name='{}-410-backup-vol'.format(self.RESOURCE_PREFIX))
-        openstack_utils.resource_reaches_status(
-            self.cinder_client.backups,
-            vol_backup.id,
-            wait_iteration_max_time=180,
-            stop_after_attempt=30,
-            expected_status='available',
-            msg='Volume status wait')
         # Delete the volume
         openstack_utils.delete_volume(self.cinder_client, cinder_vol.id)
         # Restore the volume
@@ -122,7 +137,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
             wait_iteration_max_time=180,
             stop_after_attempt=15,
             expected_status='available',
-            msg='Backup status wait')
+            msg='Restored backup volume')
         # Delete the backup
         openstack_utils.delete_volume_backup(
             self.cinder_client,
@@ -143,12 +158,12 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
         obj_count_samples.append(obj_count)
         pool_size_samples.append(kb_used)
 
-        name = '{}-410-vol'.format(self.RESOURCE_PREFIX)
         vols = self.cinder_client.volumes.list()
         try:
-            cinder_vols = [v for v in vols if v.name == name]
+            cinder_vols = [v for v in vols if v.name == cinder_vol_name]
         except AttributeError:
-            cinder_vols = [v for v in vols if v.display_name == name]
+            cinder_vols = [v for v in vols if
+                           v.display_name == cinder_vol_name]
         if not cinder_vols:
             # NOTE(hopem): it appears that at some point cinder-backup stopped
             # restoring volume metadata properly so revert to default name if

zaza/openstack/charm_tests/glance/setup.py

@@ -32,8 +32,37 @@ def basic_setup():
     """
 
 
+def _get_default_glance_client():
+    """Create default Glance client using overcloud credentials."""
+    keystone_session = openstack_utils.get_overcloud_keystone_session()
+    glance_client = openstack_utils.get_glance_session_client(keystone_session)
+    return glance_client
+
+
+def get_stores_info(glance_client=None):
+    """Retrieve glance backing store info.
+
+    :param glance_client: Authenticated glanceclient
+    :type glance_client: glanceclient.Client
+    """
+    glance_client = glance_client or _get_default_glance_client()
+    stores = glance_client.images.get_stores_info().get("stores", [])
+    return stores
+
+
+def get_store_ids(glance_client=None):
+    """Retrieve glance backing store ids.
+
+    :param glance_client: Authenticated glanceclient
+    :type glance_client: glanceclient.Client
+    """
+    stores = get_stores_info(glance_client)
+    return [store["id"] for store in stores]
+
+
 def add_image(image_url, glance_client=None, image_name=None, tags=[],
-              properties=None):
+              properties=None, backend=None, disk_format='qcow2',
+              visibility='public', container_format='bare'):
     """Retrieve image from ``image_url`` and add it to glance.
 
     :param image_url: Retrievable URL with image data
@@ -47,10 +76,14 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[],
     :param properties: Properties to add to image
     :type properties: dict
     """
-    if not glance_client:
-        keystone_session = openstack_utils.get_overcloud_keystone_session()
-        glance_client = openstack_utils.get_glance_session_client(
-            keystone_session)
+    glance_client = glance_client or _get_default_glance_client()
+    if backend is not None:
+        stores = get_store_ids(glance_client)
+        if backend not in stores:
+            raise ValueError("Invalid backend: %(backend)s "
+                             "(available: %(available)s)" % {
+                                 "backend": backend,
+                                 "available": ", ".join(stores)})
     if image_name:
         image = openstack_utils.get_images_by_name(
             glance_client, image_name)
@@ -65,7 +98,11 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[],
             image_url,
             image_name,
             tags=tags,
-            properties=properties)
+            properties=properties,
+            backend=backend,
+            disk_format=disk_format,
+            visibility=visibility,
+            container_format=container_format)
 
 
 def add_cirros_image(glance_client=None, image_name=None):

zaza/openstack/charm_tests/heat/tests.py

@@ -74,13 +74,9 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest):
     def test_400_heat_resource_types_list(self):
         """Check default resource list behavior and confirm functionality."""
         logging.info('Checking default heat resource list...')
-        try:
-            types = self.heat_client.resource_types.list()
-            self.assertIsInstance(types, list, "Resource type is not a list!")
-            self.assertGreater(len(types), 0, "Resource type list len is zero")
-        except Exception as e:
-            msg = 'Resource type list failed: {}'.format(e)
-            self.fail(msg)
+        types = self.heat_client.resource_types.list()
+        self.assertIsInstance(types, list, "Resource type is not a list!")
+        self.assertGreater(len(types), 0, "Resource type list len is zero")
 
     def test_410_heat_stack_create_delete(self):
         """Create stack, confirm nova compute resource, delete stack."""

zaza/openstack/charm_tests/ironic/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Canonical Ltd.
+# Copyright 2020 Canonical Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Test security checklist."""
+"""Collection of code for setting up and testing ironic."""

zaza/openstack/charm_tests/ironic/setup.py

@@ -0,0 +1,184 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Code for configuring ironic."""
+
+import copy
+import os
+import tenacity
+
+import zaza.openstack.charm_tests.glance.setup as glance_setup
+import zaza.openstack.utilities.openstack as openstack_utils
+from zaza.openstack.utilities import (
+    cli as cli_utils,
+)
+import zaza.model as zaza_model
+
+
+FLAVORS = {
+    'bm1.small': {
+        'flavorid': 2,
+        'ram': 2048,
+        'disk': 20,
+        'vcpus': 1,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL1_SMALL": 1,
+        },
+    },
+    'bm1.medium': {
+        'flavorid': 3,
+        'ram': 4096,
+        'disk': 40,
+        'vcpus': 2,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL1_MEDIUM": 1,
+        },
+    },
+    'bm1.large': {
+        'flavorid': 4,
+        'ram': 8192,
+        'disk': 40,
+        'vcpus': 4,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL1_LARGE": 1,
+        },
+    },
+    'bm1.tempest': {
+        'flavorid': 6,
+        'ram': 256,
+        'disk': 1,
+        'vcpus': 1,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL1_TEMPEST": 1,
+        },
+    },
+    'bm2.tempest': {
+        'flavorid': 7,
+        'ram': 512,
+        'disk': 1,
+        'vcpus': 1,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL2_TEMPEST": 1,
+        },
+    },
+}
+
+
+def _add_image(url, image_name, backend="swift",
+               disk_format="raw", container_format="bare"):
+    for attempt in tenacity.Retrying(
+            stop=tenacity.stop_after_attempt(3),
+            reraise=True):
+        with attempt:
+            glance_setup.add_image(
+                url,
+                image_name=image_name,
+                backend=backend,
+                disk_format=disk_format,
+                container_format=container_format)
+
+
+def add_ironic_deployment_image(initrd_url=None, kernel_url=None):
+    """Add Ironic deploy images to glance.
+
+    :param initrd_url: URL where the ari image resides
+    :type initrd_url: str
+    :param kernel_url: URL where the aki image resides
+    :type kernel_url: str
+    """
+    base_name = 'ironic-deploy'
+    initrd_name = "{}-initrd".format(base_name)
+    vmlinuz_name = "{}-vmlinuz".format(base_name)
+    if not initrd_url:
+        initrd_url = os.environ.get('TEST_IRONIC_DEPLOY_INITRD', None)
+    if not kernel_url:
+        kernel_url = os.environ.get('TEST_IRONIC_DEPLOY_VMLINUZ', None)
+    if not all([initrd_url, kernel_url]):
+        raise ValueError("Missing required deployment image URLs")
+
+    _add_image(
+        initrd_url,
+        initrd_name,
+        backend="swift",
+        disk_format="ari",
+        container_format="ari")
+
+    _add_image(
+        kernel_url,
+        vmlinuz_name,
+        backend="swift",
+        disk_format="aki",
+        container_format="aki")
+
+
+def add_ironic_os_image(image_url=None):
+    """Upload the operating system images built for bare metal deployments.
+
+    :param image_url: URL where the image resides
+    :type image_url: str
+    """
+    image_url = image_url or os.environ.get(
+        'TEST_IRONIC_RAW_BM_IMAGE', None)
+    image_name = "baremetal-ubuntu-image"
+    if image_url is None:
+        raise ValueError("Missing image_url")
+
+    _add_image(
+        image_url,
+        image_name,
+        backend="swift",
+        disk_format="raw",
+        container_format="bare")
+
+
+def set_temp_url_secret():
+    """Run the set-temp-url-secret on the ironic-conductor leader.
+
+    This is needed if direct boot method is enabled.
+    """
+    zaza_model.run_action_on_leader(
+        'ironic-conductor',
+        'set-temp-url-secret',
+        action_params={})
+
+
+def create_bm_flavors(nova_client=None):
+    """Create baremetal flavors.
+
+    :param nova_client: Authenticated nova client
+    :type nova_client: novaclient.v2.client.Client
+    """
+    if not nova_client:
+        keystone_session = openstack_utils.get_overcloud_keystone_session()
+        nova_client = openstack_utils.get_nova_session_client(
+            keystone_session)
+    cli_utils.setup_logging()
+    names = [flavor.name for flavor in nova_client.flavors.list()]
+    # Disable scheduling based on standard flavor properties
+    default_properties = {
+        "resources:VCPU": 0,
+        "resources:MEMORY_MB": 0,
+        "resources:DISK_GB": 0,
+    }
+    for flavor in FLAVORS.keys():
+        if flavor not in names:
+            properties = copy.deepcopy(default_properties)
+            properties.update(FLAVORS[flavor]["properties"])
+            bm_flavor = nova_client.flavors.create(
+                name=flavor,
+                ram=FLAVORS[flavor]['ram'],
+                vcpus=FLAVORS[flavor]['vcpus'],
+                disk=FLAVORS[flavor]['disk'],
+                flavorid=FLAVORS[flavor]['flavorid'])
+            bm_flavor.set_keys(properties)

zaza/openstack/charm_tests/ironic/tests.py

@@ -0,0 +1,83 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Encapsulate ironic testing."""
+
+import logging
+
+import ironicclient.client as ironic_client
+import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.openstack.utilities.openstack as openstack_utils
+
+
+def _get_ironic_client(ironic_api_version="1.58"):
+    keystone_session = openstack_utils.get_overcloud_keystone_session()
+    ironic = ironic_client.Client(1, session=keystone_session,
+                                  os_ironic_api_version=ironic_api_version)
+    return ironic
+
+
+class IronicTest(test_utils.OpenStackBaseTest):
+    """Run Ironic specific tests."""
+
+    _SERVICES = ['ironic-api']
+
+    def test_110_catalog_endpoints(self):
+        """Verify that the endpoints are present in the catalog."""
+        overcloud_auth = openstack_utils.get_overcloud_auth()
+        keystone_client = openstack_utils.get_keystone_client(
+            overcloud_auth)
+        actual_endpoints = keystone_client.service_catalog.get_endpoints()
+        actual_interfaces = [endpoint['interface'] for endpoint in
+                             actual_endpoints["baremetal"]]
+        for expected_interface in ('internal', 'admin', 'public'):
+            assert(expected_interface in actual_interfaces)
+
+    def test_400_api_connection(self):
+        """Simple api calls to check service is up and responding."""
+        ironic = _get_ironic_client()
+
+        logging.info('listing conductors')
+        conductors = ironic.conductor.list()
+        assert(len(conductors) > 0)
+
+        # By default, only IPMI HW type is enabled. iDrac and Redfish
+        # can optionally be enabled
+        drivers = ironic.driver.list()
+        driver_names = [drv.name for drv in drivers]
+
+        expected = ['intel-ipmi', 'ipmi']
+        for exp in expected:
+            assert(exp in driver_names)
+        assert(len(driver_names) == 2)
+
+    def test_900_restart_on_config_change(self):
+        """Checking restart happens on config change.
+
+        Change debug mode and assert that change propagates to the correct
+        file and that services are restarted as a result
+        """
+        self.restart_on_changed_debug_oslo_config_file(
+            '/etc/ironic/ironic.conf', self._SERVICES)
+
+    def test_910_pause_resume(self):
+        """Run pause and resume tests.
+
+        Pause service and check services are stopped then resume and check
+        they are started
+        """
+        logging.info('Skipping pause resume test LP: #1886202...')
+        return
+        with self.pause_resume(self._SERVICES):
+            logging.info("Testing pause resume")

zaza/openstack/charm_tests/kerberos/setup.py

@@ -113,6 +113,9 @@ def retrieve_and_attach_keytab():
                                    'keystone_keytab',
                                    tmp_file)
 
+    zaza.model.wait_for_application_states()
+    zaza.model.block_until_all_units_idle()
+
 
 def openstack_setup_kerberos():
     """Create a test domain, project, and user for kerberos tests."""

zaza/openstack/charm_tests/keystone/tests.py

@@ -380,7 +380,12 @@ class SecurityTests(BaseKeystoneTest):
 
 
 class LdapTests(BaseKeystoneTest):
-    """Keystone ldap tests tests."""
+    """Keystone ldap tests."""
+
+    non_string_type_keys = ('ldap-user-enabled-mask',
+                            'ldap-user-enabled-invert',
+                            'ldap-group-members-are-ids',
+                            'ldap-use-pool')
 
     @classmethod
     def setUpClass(cls):
@@ -434,24 +439,124 @@ class LdapTests(BaseKeystoneTest):
     def test_100_keystone_ldap_users(self):
         """Validate basic functionality of keystone API with ldap."""
         application_name = 'keystone-ldap'
-        config = self._get_ldap_config()
+        intended_cfg = self._get_ldap_config()
+        current_cfg, non_string_cfg = (
+            self.config_current_separate_non_string_type_keys(
+                self.non_string_type_keys, intended_cfg, application_name)
+        )
 
         with self.config_change(
-                self.config_current(application_name, config.keys()),
-                config,
-                application_name=application_name):
-            logging.info(
-                'Waiting for users to become available in keystone...'
-            )
-            test_config = lifecycle_utils.get_charm_config(fatal=False)
-            zaza.model.wait_for_application_states(
-                states=test_config.get("target_deploy_status", {})
-            )
+                {},
+                non_string_cfg,
+                application_name=application_name,
+                reset_to_charm_default=True):
+            with self.config_change(
+                    current_cfg,
+                    intended_cfg,
+                    application_name=application_name):
+                logging.info(
+                    'Waiting for users to become available in keystone...'
+                )
+                test_config = lifecycle_utils.get_charm_config(fatal=False)
+                zaza.model.wait_for_application_states(
+                    states=test_config.get("target_deploy_status", {})
+                )
 
-            with self.v3_keystone_preferred():
-                # NOTE(jamespage): Test fixture should have johndoe and janedoe
-                #                  accounts
-                johndoe = self._find_keystone_v3_user('john doe', 'userdomain')
-                self.assertIsNotNone(johndoe, "user 'john doe' was unknown")
-                janedoe = self._find_keystone_v3_user('jane doe', 'userdomain')
-                self.assertIsNotNone(janedoe, "user 'jane doe' was unknown")
+                with self.v3_keystone_preferred():
+                    # NOTE(jamespage): Test fixture should have
+                    #                  johndoe and janedoe accounts
+                    johndoe = self._find_keystone_v3_user(
+                        'john doe', 'userdomain')
+                    self.assertIsNotNone(
+                        johndoe, "user 'john doe' was unknown")
+                    janedoe = self._find_keystone_v3_user(
+                        'jane doe', 'userdomain')
+                    self.assertIsNotNone(
+                        janedoe, "user 'jane doe' was unknown")
+
+
+class LdapExplicitCharmConfigTests(LdapTests):
+    """Keystone ldap tests."""
+
+    def _get_ldap_config(self):
+        """Generate ldap config for current model.
+
+        :return: tuple of whether ldap-server is running and if so, config
+            for the keystone-ldap application.
+        :rtype: Tuple[bool, Dict[str,str]]
+        """
+        ldap_ips = zaza.model.get_app_ips("ldap-server")
+        self.assertTrue(ldap_ips, "Should be at least one ldap server")
+        return {
+            'ldap-server': "ldap://{}".format(ldap_ips[0]),
+            'ldap-user': 'cn=admin,dc=test,dc=com',
+            'ldap-password': 'crapper',
+            'ldap-suffix': 'dc=test,dc=com',
+            'domain-name': 'userdomain',
+            'ldap-query-scope': 'one',
+            'ldap-user-objectclass': 'inetOrgPerson',
+            'ldap-user-id-attribute': 'cn',
+            'ldap-user-name-attribute': 'sn',
+            'ldap-user-enabled-attribute': 'enabled',
+            'ldap-user-enabled-invert': False,
+            'ldap-user-enabled-mask': 0,
+            'ldap-user-enabled-default': 'True',
+            'ldap-group-tree-dn': 'ou=groups',
+            'ldap-group-objectclass': '',
+            'ldap-group-id-attribute': 'cn',
+            'ldap-group-member-attribute': 'memberUid',
+            'ldap-group-members-are-ids': True,
+            'ldap-config-flags': '{group_objectclass: "posixGroup",'
+                                 ' use_pool: True,'
+                                 ' group_tree_dn: "group_tree_dn_foobar"}',
+        }
+
+    def test_200_config_flags_precedence(self):
+        """Validates precedence when the same config options are used."""
+        application_name = 'keystone-ldap'
+        intended_cfg = self._get_ldap_config()
+        current_cfg, non_string_cfg = (
+            self.config_current_separate_non_string_type_keys(
+                self.non_string_type_keys, intended_cfg, application_name)
+        )
+
+        with self.config_change(
+                {},
+                non_string_cfg,
+                application_name=application_name,
+                reset_to_charm_default=True):
+            with self.config_change(
+                    current_cfg,
+                    intended_cfg,
+                    application_name=application_name):
+                logging.info(
+                    'Performing LDAP settings validation in keystone.conf...'
+                )
+                test_config = lifecycle_utils.get_charm_config(fatal=False)
+                zaza.model.wait_for_application_states(
+                    states=test_config.get("target_deploy_status", {})
+                )
+                units = zaza.model.get_units("keystone-ldap",
+                                             model_name=self.model_name)
+                result = zaza.model.run_on_unit(
+                    units[0].name,
+                    "cat /etc/keystone/domains/keystone.userdomain.conf")
+                # not present in charm config, but present in config flags
+                self.assertIn("use_pool = True", result['stdout'],
+                              "use_pool value is expected to be present and "
+                              "set to True in the config file")
+                # ldap-config-flags overriding empty charm config value
+                self.assertIn("group_objectclass = posixGroup",
+                              result['stdout'],
+                              "group_objectclass is expected to be present and"
+                              " set to posixGroup in the config file")
+                # overridden by charm config, not written to file
+                self.assertNotIn(
+                    "group_tree_dn_foobar",
+                    result['stdout'],
+                    "user_tree_dn ldap-config-flags value needs to be "
+                    "overridden by ldap-user-tree-dn in config file")
+                # complementing the above, value used is from charm setting
+                self.assertIn("group_tree_dn = ou=groups", result['stdout'],
+                              "user_tree_dn value is expected to be present "
+                              "and set to dc=test,dc=com in the config file")

zaza/openstack/charm_tests/magpie/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Collection of code for setting up and testing Magpie."""

zaza/openstack/charm_tests/magpie/tests.py

@@ -0,0 +1,82 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Encapsulate Magpie testing."""
+
+import logging
+
+import zaza
+
+import zaza.model
+import zaza.openstack.charm_tests.test_utils as test_utils
+
+
+class MagpieTest(test_utils.BaseCharmTest):
+    """Base Magpie tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for Magpie charm operation tests."""
+        super(MagpieTest, cls).setUpClass()
+        unit_names = sorted(
+            [i.entity_id
+             for i in zaza.model.get_units('magpie')])
+        cls.test_unit_0 = unit_names[0]
+        cls.test_unit_1 = unit_names[1]
+
+    def test_break_dns_single(self):
+        """Check DNS failure is reflected in workload status."""
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            'mv /etc/resolv.conf /etc/resolv.conf.bak')
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            './hooks/update-status')
+        zaza.model.block_until_unit_wl_message_match(
+            self.test_unit_0,
+            '.*rev dns failed.*')
+        logging.info('Restoring /etc/resolv.conf')
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            'mv /etc/resolv.conf.bak /etc/resolv.conf')
+        logging.info('Updating status')
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            './hooks/update-status')
+
+    def test_break_ping_single(self):
+        """Check ping failure is reflected in workload status."""
+        icmp = "iptables {} INPUT -p icmp --icmp-type echo-request -j REJECT"
+        logging.info('Blocking ping on {}'.format(self.test_unit_1))
+        zaza.model.run_on_unit(
+            self.test_unit_1,
+            icmp.format('--append'))
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            './hooks/update-status')
+        logging.info('Checking status on {}'.format(self.test_unit_0))
+        zaza.model.block_until_unit_wl_message_match(
+            self.test_unit_0,
+            '.*icmp failed.*')
+        logging.info('Allowing ping on {}'.format(self.test_unit_1))
+        zaza.model.run_on_unit(
+            self.test_unit_1,
+            icmp.format('--delete'))
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            './hooks/update-status')
+        logging.info('Checking status on {}'.format(self.test_unit_0))
+        zaza.model.block_until_unit_wl_message_match(
+            self.test_unit_0,
+            '.*icmp ok.*')

zaza/openstack/charm_tests/manila_ganesha/tests.py

@@ -20,7 +20,6 @@ from tenacity import Retrying, stop_after_attempt, wait_exponential
 
 from manilaclient import client as manilaclient
 
-import zaza.openstack.charm_tests.glance.setup as glance_setup
 import zaza.openstack.charm_tests.neutron.tests as neutron_tests
 import zaza.openstack.charm_tests.nova.utils as nova_utils
 import zaza.openstack.charm_tests.test_utils as test_utils
@@ -62,13 +61,7 @@ packages:
             share_proto="nfs", size=1)
 
         # Spawn Servers
-        instance_1 = guest.launch_instance(
-            glance_setup.LTS_IMAGE_NAME,
-            vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX),
-            userdata=self.INSTANCE_USERDATA)
-        instance_2 = guest.launch_instance(
-            glance_setup.LTS_IMAGE_NAME,
-            vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX),
+        instance_1, instance_2 = self.launch_guests(
             userdata=self.INSTANCE_USERDATA)
 
         fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0]

zaza/openstack/charm_tests/neutron/setup.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 # Copyright 2018 Canonical Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,6 +26,8 @@ from zaza.openstack.utilities import (
     openstack as openstack_utils,
 )
 
+import zaza.charm_lifecycle.utils as lifecycle_utils
+
 
 # The overcloud network configuration settings are declared.
 # These are the network configuration settings under test.
@@ -81,12 +81,20 @@ def basic_overcloud_network(limit_gws=None):
     # Get keystone session
     keystone_session = openstack_utils.get_overcloud_keystone_session()
 
+    # Get optional use_juju_wait for netw ork option
+    options = (lifecycle_utils
+               .get_charm_config(fatal=False)
+               .get('configure_options', {}))
+    use_juju_wait = options.get(
+        'configure_gateway_ext_port_use_juju_wait', True)
+
     # Handle network for OpenStack-on-OpenStack scenarios
     if juju_utils.get_provider_type() == "openstack":
         undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session()
         network.setup_gateway_ext_port(network_config,
                                        keystone_session=undercloud_ks_sess,
-                                       limit_gws=None)
+                                       limit_gws=None,
+                                       use_juju_wait=use_juju_wait)
 
     # Confugre the overcloud network
     network.setup_sdn(network_config, keystone_session=keystone_session)

zaza/openstack/charm_tests/neutron/tests.py

@@ -24,6 +24,8 @@ import copy
 import logging
 import tenacity
 
+from neutronclient.common import exceptions as neutronexceptions
+
 import zaza
 import zaza.openstack.charm_tests.nova.utils as nova_utils
 import zaza.openstack.charm_tests.test_utils as test_utils
@@ -635,7 +637,8 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest):
     def validate_instance_can_reach_other(self,
                                           instance_1,
                                           instance_2,
-                                          verify):
+                                          verify,
+                                          mtu=None):
         """
         Validate that an instance can reach a fixed and floating of another.
 
@@ -644,6 +647,12 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest):
 
         :param instance_2: The instance to check networking from
         :type instance_2: nova_client.Server
+
+        :param verify: callback to verify result
+        :type verify: callable
+
+        :param mtu: Check that we can send non-fragmented packets of given size
+        :type mtu: Optional[int]
         """
         floating_1 = floating_ips_from_instance(instance_1)[0]
         floating_2 = floating_ips_from_instance(instance_2)[0]
@@ -653,19 +662,30 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest):
         password = guest.boot_tests['bionic'].get('password')
         privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)
 
-        openstack_utils.ssh_command(
-            username, floating_1, 'instance-1',
-            'ping -c 1 {}'.format(address_2),
-            password=password, privkey=privkey, verify=verify)
+        cmds = [
+            'ping -c 1',
+        ]
+        if mtu:
+            # the on-wire packet will be 28 bytes larger than the value
+            # provided to ping(8) -s parameter
+            packetsize = mtu - 28
+            cmds.append(
+                'ping -M do -s {} -c 1'.format(packetsize))
 
-        openstack_utils.ssh_command(
-            username, floating_1, 'instance-1',
-            'ping -c 1 {}'.format(floating_2),
-            password=password, privkey=privkey, verify=verify)
+        for cmd in cmds:
+            openstack_utils.ssh_command(
+                username, floating_1, 'instance-1',
+                '{} {}'.format(cmd, address_2),
+                password=password, privkey=privkey, verify=verify)
+
+            openstack_utils.ssh_command(
+                username, floating_1, 'instance-1',
+                '{} {}'.format(cmd, floating_2),
+                password=password, privkey=privkey, verify=verify)
 
     @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
                     reraise=True, stop=tenacity.stop_after_attempt(8))
-    def validate_instance_can_reach_router(self, instance, verify):
+    def validate_instance_can_reach_router(self, instance, verify, mtu=None):
         """
         Validate that an instance can reach it's primary gateway.
 
@@ -676,6 +696,12 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest):
 
         :param instance: The instance to check networking from
         :type instance: nova_client.Server
+
+        :param verify: callback to verify result
+        :type verify: callable
+
+        :param mtu: Check that we can send non-fragmented packets of given size
+        :type mtu: Optional[int]
         """
         address = floating_ips_from_instance(instance)[0]
 
@@ -683,9 +709,20 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest):
         password = guest.boot_tests['bionic'].get('password')
         privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)
 
-        openstack_utils.ssh_command(
-            username, address, 'instance', 'ping -c 1 192.168.0.1',
-            password=password, privkey=privkey, verify=verify)
+        cmds = [
+            'ping -c 1',
+        ]
+        if mtu:
+            # the on-wire packet will be 28 bytes larger than the value
+            # provided to ping(8) -s parameter
+            packetsize = mtu - 28
+            cmds.append(
+                'ping -M do -s {} -c 1'.format(packetsize))
+
+        for cmd in cmds:
+            openstack_utils.ssh_command(
+                username, address, 'instance', '{} 192.168.0.1'.format(cmd),
+                password=password, privkey=privkey, verify=verify)
 
     @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60),
                     reraise=True, stop=tenacity.stop_after_attempt(8),
@@ -726,21 +763,67 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest):
                 assert agent['admin_state_up']
                 assert agent['alive']
 
+    def effective_network_mtu(self, network_name):
+        """Retrieve effective MTU for a network.
+
+        If the `instance-mtu` configuration option is set to a value lower than
+        the network MTU this method will return the value of that. Otherwise
+        Neutron's value for MTU on a network will be returned.
+
+        :param network_name: Name of network to query
+        :type network_name: str
+        :returns: MTU for network
+        :rtype: int
+        """
+        cfg_instance_mtu = None
+        for app in ('neutron-gateway', 'neutron-openvswitch'):
+            try:
+                cfg = zaza.model.get_application_config(app)
+                cfg_instance_mtu = int(cfg['instance-mtu']['value'])
+                break
+            except KeyError:
+                pass
+
+        networks = self.neutron_client.show_network('', name=network_name)
+        network_mtu = int(next(iter(networks['networks']))['mtu'])
+
+        if cfg_instance_mtu and cfg_instance_mtu < network_mtu:
+            logging.info('Using MTU from application "{}" config: {}'
+                         .format(app, cfg_instance_mtu))
+            return cfg_instance_mtu
+        else:
+            logging.info('Using MTU from network "{}": {}'
+                         .format(network_name, network_mtu))
+            return network_mtu
+
     def check_connectivity(self, instance_1, instance_2):
         """Run North/South and East/West connectivity tests."""
         def verify(stdin, stdout, stderr):
             """Validate that the SSH command exited 0."""
             self.assertEqual(stdout.channel.recv_exit_status(), 0)
 
+        try:
+            mtu_1 = self.effective_network_mtu(
+                network_name_from_instance(instance_1))
+            mtu_2 = self.effective_network_mtu(
+                network_name_from_instance(instance_2))
+            mtu_min = min(mtu_1, mtu_2)
+        except neutronexceptions.NotFound:
+            # Older versions of OpenStack cannot look up network by name, just
+            # skip the check if that is the case.
+            mtu_1 = mtu_2 = mtu_min = None
+
         # Verify network from 1 to 2
-        self.validate_instance_can_reach_other(instance_1, instance_2, verify)
+        self.validate_instance_can_reach_other(
+            instance_1, instance_2, verify, mtu_min)
 
         # Verify network from 2 to 1
-        self.validate_instance_can_reach_other(instance_2, instance_1, verify)
+        self.validate_instance_can_reach_other(
+            instance_2, instance_1, verify, mtu_min)
 
         # Validate tenant to external network routing
-        self.validate_instance_can_reach_router(instance_1, verify)
-        self.validate_instance_can_reach_router(instance_2, verify)
+        self.validate_instance_can_reach_router(instance_1, verify, mtu_1)
+        self.validate_instance_can_reach_router(instance_2, verify, mtu_2)
 
 
 def floating_ips_from_instance(instance):
@@ -769,6 +852,17 @@ def fixed_ips_from_instance(instance):
     return ips_from_instance(instance, 'fixed')
 
 
+def network_name_from_instance(instance):
+    """Retrieve name of primary network the instance is attached to.
+
+    :param instance: The instance to fetch name of network from.
+    :type instance: nova_client.Server
+    :returns: Name of primary network the instance is attached to.
+    :rtype: str
+    """
+    return next(iter(instance.addresses))
+
+
 def ips_from_instance(instance, ip_type):
     """
     Retrieve IPs of a certain type from an instance.
@@ -786,7 +880,8 @@ def ips_from_instance(instance, ip_type):
             "Only 'floating' and 'fixed' are valid IP types to search for"
         )
     return list([
-        ip['addr'] for ip in instance.addresses['private']
+        ip['addr'] for ip in instance.addresses[
+            network_name_from_instance(instance)]
         if ip['OS-EXT-IPS:type'] == ip_type])
 
 
@@ -794,11 +889,22 @@ class NeutronNetworkingTest(NeutronNetworkingBase):
     """Ensure that openstack instances have valid networking."""
 
     def test_instances_have_networking(self):
-        """Validate North/South and East/West networking."""
-        self.launch_guests()
+        """Validate North/South and East/West networking.
+
+        Tear down can optionally be disabled by setting the module path +
+        class name + run_tearDown key under the `tests_options` key in
+        tests.yaml.
+
+        Abbreviated example:
+        ...charm_tests.neutron.tests.NeutronNetworkingTest.run_tearDown: false
+        """
         instance_1, instance_2 = self.retrieve_guests()
+        if not all([instance_1, instance_2]):
+            self.launch_guests()
+            instance_1, instance_2 = self.retrieve_guests()
         self.check_connectivity(instance_1, instance_2)
-        self.run_resource_cleanup = True
+        self.run_resource_cleanup = self.get_my_tests_options(
+            'run_resource_cleanup', True)
 
 
 class NeutronNetworkingVRRPTests(NeutronNetworkingBase):

zaza/openstack/charm_tests/neutron_arista/setup.py

@@ -45,9 +45,15 @@ def download_arista_image():
         if os.environ['TEST_ARISTA_IMAGE_REMOTE']:
             logging.info('Downloading Arista image from {}'
                          .format(os.environ['TEST_ARISTA_IMAGE_REMOTE']))
-            openstack_utils.download_image(
-                os.environ['TEST_ARISTA_IMAGE_REMOTE'],
-                os.environ['TEST_ARISTA_IMAGE_LOCAL'])
+
+            for attempt in tenacity.Retrying(
+                    stop=tenacity.stop_after_attempt(3),
+                    reraise=True):
+                with attempt:
+                    openstack_utils.download_image(
+                        os.environ['TEST_ARISTA_IMAGE_REMOTE'],
+                        os.environ['TEST_ARISTA_IMAGE_LOCAL'])
+
     except KeyError:
         # TEST_ARISTA_IMAGE_REMOTE isn't set, which means the image is already
         # available at TEST_ARISTA_IMAGE_LOCAL

zaza/openstack/charm_tests/nova/tests.py

@@ -38,31 +38,34 @@ class BaseGuestCreateTest(unittest.TestCase):
         zaza.openstack.configure.guest.launch_instance(instance_key)
 
 
-class CirrosGuestCreateTest(BaseGuestCreateTest):
+class CirrosGuestCreateTest(test_utils.OpenStackBaseTest):
     """Tests to launch a cirros image."""
 
     def test_launch_small_instance(self):
         """Launch a cirros instance and test connectivity."""
-        zaza.openstack.configure.guest.launch_instance(
-            glance_setup.CIRROS_IMAGE_NAME)
+        self.RESOURCE_PREFIX = 'zaza-nova'
+        self.launch_guest(
+            'cirros', instance_key=glance_setup.CIRROS_IMAGE_NAME)
 
 
-class LTSGuestCreateTest(BaseGuestCreateTest):
+class LTSGuestCreateTest(test_utils.OpenStackBaseTest):
     """Tests to launch a LTS image."""
 
     def test_launch_small_instance(self):
         """Launch a Bionic instance and test connectivity."""
-        zaza.openstack.configure.guest.launch_instance(
-            glance_setup.LTS_IMAGE_NAME)
+        self.RESOURCE_PREFIX = 'zaza-nova'
+        self.launch_guest(
+            'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME)
 
 
-class LTSGuestCreateVolumeBackedTest(BaseGuestCreateTest):
+class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest):
     """Tests to launch a LTS image."""
 
     def test_launch_small_instance(self):
         """Launch a Bionic instance and test connectivity."""
-        zaza.openstack.configure.guest.launch_instance(
-            glance_setup.LTS_IMAGE_NAME,
+        self.RESOURCE_PREFIX = 'zaza-nova'
+        self.launch_guest(
+            'volume-backed-ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME,
             use_boot_volume=True)
 
 

zaza/openstack/charm_tests/octavia/diskimage_retrofit/setup.py

@@ -15,6 +15,7 @@
 """Code for configuring octavia-diskimage-retrofit."""
 
 import logging
+import tenacity
 
 import zaza.model
 
@@ -39,12 +40,20 @@ def retrofit_amphora_image(unit='octavia-diskimage-retrofit/0',
     if image_id:
         params.update({'source-image': image_id})
 
-    # NOTE(fnordahl) ``zaza.model.run_action_on_leader`` fails here,
-    # apparently has to do with handling of subordinates in ``libjuju`` or
-    # ``juju`` itself.
-    action = zaza.model.run_action(
-        unit,
-        'retrofit-image',
-        action_params=params,
-        raise_on_failure=True)
+    # NOTE(fnordahl) the retrofit process involves downloading packages from
+    # the internet and is as such susceptible to random failures due to
+    # internet gremlins.
+    for attempt in tenacity.Retrying(
+            stop=tenacity.stop_after_attempt(3),
+            wait=tenacity.wait_exponential(
+            multiplier=1, min=2, max=10)):
+        with attempt:
+            # NOTE(fnordahl) ``zaza.model.run_action_on_leader`` fails here,
+            # apparently has to do with handling of subordinates in ``libjuju``
+            # or ``juju`` itself.
+            action = zaza.model.run_action(
+                unit,
+                'retrofit-image',
+                action_params=params,
+                raise_on_failure=True)
     return action

zaza/openstack/charm_tests/octavia/tests.py

@@ -18,6 +18,8 @@ import logging
 import subprocess
 import tenacity
 
+from keystoneauth1 import exceptions as keystone_exceptions
+import octaviaclient.api.v2.octavia
 import osc_lib.exceptions
 
 import zaza.openstack.charm_tests.test_utils as test_utils
@@ -80,18 +82,62 @@ class LBAASv2Test(test_utils.OpenStackBaseTest):
         # List of floating IPs created by this test
         cls.fips = []
 
-    def resource_cleanup(self):
-        """Remove resources created during test execution."""
+    def _remove_amphorae_instances(self):
+        """Remove amphorae instances forcefully.
+
+        In some situations Octavia is unable to remove load balancer resources.
+        This helper can be used to remove the underlying instances.
+        """
+        result = self.octavia_client.amphora_list()
+        for amphora in result.get('amphorae', []):
+            for server in self.nova_client.servers.list():
+                if 'compute_id' in amphora and server.id == amphora[
+                        'compute_id']:
+                    try:
+                        openstack_utils.delete_resource(
+                            self.nova_client.servers,
+                            server.id,
+                            msg="server")
+                    except AssertionError as e:
+                        logging.warning(
+                            'Gave up waiting for resource cleanup: "{}"'
+                            .format(str(e)))
+
+    @tenacity.retry(stop=tenacity.stop_after_attempt(3),
+                    wait=tenacity.wait_exponential(
+                        multiplier=1, min=2, max=10))
+    def resource_cleanup(self, only_local=False):
+        """Remove resources created during test execution.
+
+        :param only_local: When set to true do not call parent method
+        :type only_local: bool
+        """
         for lb in self.loadbalancers:
-            self.octavia_client.load_balancer_delete(lb['id'], cascade=True)
             try:
-                self.wait_for_lb_resource(
-                    self.octavia_client.load_balancer_show, lb['id'],
-                    provisioning_status='DELETED')
-            except osc_lib.exceptions.NotFound:
-                pass
+                self.octavia_client.load_balancer_delete(
+                    lb['id'], cascade=True)
+            except octaviaclient.api.v2.octavia.OctaviaClientException as e:
+                logging.info('Octavia is unable to delete load balancer: "{}"'
+                             .format(e))
+                logging.info('Attempting to forcefully remove amphorae')
+                self._remove_amphorae_instances()
+            else:
+                try:
+                    self.wait_for_lb_resource(
+                        self.octavia_client.load_balancer_show, lb['id'],
+                        provisioning_status='DELETED')
+                except osc_lib.exceptions.NotFound:
+                    pass
+        # allow resource cleanup to be run multiple times
+        self.loadbalancers = []
         for fip in self.fips:
             self.neutron_client.delete_floatingip(fip)
+        # allow resource cleanup to be run multiple times
+        self.fips = []
+
+        if only_local:
+            return
+
         # we run the parent resource_cleanup last as it will remove instances
         # referenced as members in the above cleaned up load balancers
         super(LBAASv2Test, self).resource_cleanup()
@@ -157,6 +203,7 @@ class LBAASv2Test(test_utils.OpenStackBaseTest):
                     'provider': provider,
                 }})
         lb = result['loadbalancer']
+        self.loadbalancers.append(lb)
         lb_id = lb['id']
 
         logging.info('Awaiting loadbalancer to reach provisioning_status '
@@ -283,10 +330,25 @@ class LBAASv2Test(test_utils.OpenStackBaseTest):
         for provider in self.get_lb_providers(self.octavia_client).keys():
             logging.info('Creating loadbalancer with provider {}'
                          .format(provider))
-            lb = self._create_lb_resources(self.octavia_client, provider,
-                                           vip_subnet_id, subnet_id,
-                                           payload_ips)
-            self.loadbalancers.append(lb)
+            final_exc = None
+            # NOTE: we cannot use tenacity here as the method we call into
+            # already uses it to wait for operations to complete.
+            for retry in range(0, 3):
+                try:
+                    lb = self._create_lb_resources(self.octavia_client,
+                                                   provider,
+                                                   vip_subnet_id,
+                                                   subnet_id,
+                                                   payload_ips)
+                    break
+                except (AssertionError,
+                        keystone_exceptions.connection.ConnectFailure) as e:
+                    logging.info('Retrying load balancer creation, last '
+                                 'failure: "{}"'.format(str(e)))
+                    self.resource_cleanup(only_local=True)
+                    final_exc = e
+            else:
+                raise final_exc
 
             lb_fp = openstack_utils.create_floating_ip(
                 self.neutron_client, 'ext_net', port={'id': lb['vip_port_id']})

zaza/openstack/charm_tests/openstack_dashboard/tests.py

@@ -46,11 +46,11 @@ class FailedAuth(AuthExceptions):
 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1,
                                                min=5, max=10),
                 reraise=True)
-def _login(dashboard_ip, domain, username, password):
+def _login(dashboard_url, domain, username, password, cafile=None):
     """Login to the website to get a session.
 
-    :param dashboard_ip: The IP address of the dashboard to log in to.
-    :type dashboard_ip: str
+    :param dashboard_url: The URL of the dashboard to log in to.
+    :type dashboard_url: str
     :param domain: the domain to login into
     :type domain: str
     :param username: the username to login as
@@ -62,11 +62,11 @@ def _login(dashboard_ip, domain, username, password):
     :rtype: (requests.sessions.Session, requests.models.Response)
     :raises: FailedAuth if the authorisation doesn't work
     """
-    auth_url = 'http://{}/horizon/auth/login/'.format(dashboard_ip)
+    auth_url = '{}/auth/login/'.format(dashboard_url)
 
     # start session, get csrftoken
     client = requests.session()
-    client.get(auth_url)
+    client.get(auth_url, verify=cafile)
 
     if 'csrftoken' in client.cookies:
         csrftoken = client.cookies['csrftoken']
@@ -117,7 +117,11 @@ def _login(dashboard_ip, domain, username, password):
         del auth['domain']
 
     logging.info('POST data: "{}"'.format(auth))
-    response = client.post(auth_url, data=auth, headers={'Referer': auth_url})
+    response = client.post(
+        auth_url,
+        data=auth,
+        headers={'Referer': auth_url},
+        verify=cafile)
 
     # NOTE(ajkavanagh) there used to be a trusty/icehouse test in the
     # amulet test, but as the zaza tests only test from trusty/mitaka
@@ -147,7 +151,7 @@ def _login(dashboard_ip, domain, username, password):
                 retry=tenacity.retry_if_exception_type(
                     http.client.RemoteDisconnected),
                 reraise=True)
-def _do_request(request):
+def _do_request(request, cafile=None):
     """Open a webpage via urlopen.
 
     :param request: A urllib request object.
@@ -156,7 +160,7 @@ def _do_request(request):
     :rtype: object
     :raises: URLError on protocol errors
     """
-    return urllib.request.urlopen(request)
+    return urllib.request.urlopen(request, cafile=cafile)
 
 
 class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
@@ -167,6 +171,13 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
         """Run class setup for running openstack dashboard charm tests."""
         super(OpenStackDashboardTests, cls).setUpClass()
         cls.application = 'openstack-dashboard'
+        cls.use_https = False
+        vault_relation = zaza_model.get_relation_id(
+            cls.application,
+            'vault',
+            remote_interface_name='certificates')
+        if vault_relation:
+            cls.use_https = True
 
     def test_050_local_settings_permissions_regression_check_lp1755027(self):
         """Assert regression check lp1755027.
@@ -291,22 +302,49 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
                         mismatches.append(msg)
         return mismatches
 
+    def get_base_url(self):
+        """Return the base url for http(s) requests.
+
+        :returns: URL
+        :rtype: str
+        """
+        vip = (zaza_model.get_application_config(self.application_name)
+               .get("vip").get("value"))
+        if vip:
+            ip = vip
+        else:
+            unit = zaza_model.get_unit_from_name(
+                zaza_model.get_lead_unit_name(self.application_name))
+            ip = unit.public_address
+
+        logging.debug("Dashboard ip is:{}".format(ip))
+        scheme = 'http'
+        if self.use_https:
+            scheme = 'https'
+        url = '{}://{}'.format(scheme, ip)
+        logging.debug("Base URL is: {}".format(url))
+        return url
+
+    def get_horizon_url(self):
+        """Return the url for acccessing horizon.
+
+        :returns: Horizon URL
+        :rtype: str
+        """
+        url = '{}/horizon'.format(self.get_base_url())
+        logging.info("Horizon URL is: {}".format(url))
+        return url
+
     def test_400_connection(self):
         """Test that dashboard responds to http request.
 
         Ported from amulet tests.
         """
         logging.info('Checking dashboard http response...')
-
-        unit = zaza_model.get_unit_from_name(
-            zaza_model.get_lead_unit_name(self.application_name))
-        logging.debug("... dashboard_ip is:{}".format(unit.public_address))
-
-        request = urllib.request.Request(
-            'http://{}/horizon'.format(unit.public_address))
+        request = urllib.request.Request(self.get_horizon_url())
         try:
             logging.info("... trying to fetch the page")
-            html = _do_request(request)
+            html = _do_request(request, cafile=self.cacert)
             logging.info("... fetched page")
         except Exception as e:
             logging.info("... exception raised was {}".format(str(e)))
@@ -319,8 +357,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
         """Validate that authentication succeeds for client log in."""
         logging.info('Checking authentication through dashboard...')
 
-        unit = zaza_model.get_unit_from_name(
-            zaza_model.get_lead_unit_name(self.application_name))
         overcloud_auth = openstack_utils.get_overcloud_auth()
         password = overcloud_auth['OS_PASSWORD'],
         logging.info("admin password is {}".format(password))
@@ -329,7 +365,12 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
         domain = 'admin_domain',
         username = 'admin',
         password = overcloud_auth['OS_PASSWORD'],
-        _login(unit.public_address, domain, username, password)
+        _login(
+            self.get_horizon_url(),
+            domain,
+            username,
+            password,
+            cafile=self.cacert)
         logging.info('OK')
 
     def test_404_connection(self):
@@ -338,22 +379,16 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
         Ported from amulet tests.
         """
         logging.info('Checking apache mod_status gets disabled.')
-
-        unit = zaza_model.get_unit_from_name(
-            zaza_model.get_lead_unit_name(self.application_name))
-        logging.debug("... dashboard_ip is: {}".format(unit.public_address))
-
         logging.debug('Maybe enabling hardening for apache...')
         _app_config = zaza_model.get_application_config(self.application_name)
         logging.info(_app_config['harden'])
 
-        request = urllib.request.Request(
-            'http://{}/server-status'.format(unit.public_address))
+        request = urllib.request.Request(self.get_horizon_url())
         with self.config_change(
                 {'harden': _app_config['harden'].get('value', '')},
                 {'harden': 'apache'}):
             try:
-                _do_request(request)
+                _do_request(request, cafile=self.cacert)
             except urllib.request.HTTPError as e:
                 # test failed if it didn't return 404
                 msg = "Apache mod_status check failed."

zaza/openstack/charm_tests/ovn/setup.py

@@ -0,0 +1,154 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Code for configuring OVN tests."""
+
+import logging
+
+import zaza
+
+import zaza.openstack.charm_tests.test_utils as test_utils
+
+
+class _OVNSetupHelper(test_utils.BaseCharmTest):
+    """Helper class to get at the common `config_change` helper."""
+
+    @staticmethod
+    def _get_instance_mtu_from_global_physnet_mtu():
+        """Calculate instance mtu from Neutron API global-physnet-mtu.
+
+        :returns: Value for instance mtu after migration.
+        :rtype: int
+        """
+        n_api_config = zaza.model.get_application_config('neutron-api')
+
+        # NOTE: we would have to adjust this calculation if we use IPv6 tunnel
+        # endpoints
+        GENEVE_ENCAP_OVERHEAD = 38
+        IP4_HEADER_SIZE = 20
+        return int(n_api_config['global-physnet-mtu']['value']) - (
+            GENEVE_ENCAP_OVERHEAD + IP4_HEADER_SIZE)
+
+    def _configure_apps(self, apps, cfg,
+                        first_match_raise_if_none_found=False):
+        """Conditionally configure a set of applications.
+
+        :param apps: Applications.
+        :type apps: Iterator[str]
+        :param cfg: Configuration to apply.
+        :type cfg: Dict[str,any]
+        :param first_match_raise_if_none_found: When set the method will
+                                                configure the first application
+                                                it finds in the model and raise
+                                                an exception if none are found.
+        :type first_match_raise_if_none_found: bool
+        :raises: RuntimeError
+        """
+        for app in apps:
+            try:
+                zaza.model.get_application(app)
+                for k, v in cfg.items():
+                    logging.info('Setting `{}` to "{}" on "{}"...'
+                                 .format(k, v, app))
+                    with self.config_change(cfg, cfg, app):
+                        # The intent here is to change the config and not
+                        # restore it. We accomplish that by passing in the same
+                        # value for default and alternate.
+                        #
+                        # The reason for using the `config_change` helper for
+                        # this is that it already deals with all the
+                        # permutations of config already being set etc and does
+                        # not get into trouble if the test bundle already has
+                        # the values we try to set.
+                        if first_match_raise_if_none_found:
+                            break
+                        else:
+                            continue
+                else:
+                    if first_match_raise_if_none_found:
+                        raise RuntimeError(
+                            'None of the expected apps ({}) are present in '
+                            'the model.'
+                            .format(apps)
+                        )
+            except KeyError:
+                pass
+
+    def configure_ngw_novs(self):
+        """Configure n-ovs and n-gw units."""
+        cfg = {
+            # To be able to have instances successfully survive the migration
+            # without communication issues we need to lower the MTU announced
+            # to instances prior to migration.
+            #
+            # NOTE: In a real world scenario the end user would configure the
+            # MTU at least 24 hrs prior to doing the migration to allow
+            # instances to reconfigure as they renew the DHCP lease.
+            #
+            # NOTE: For classic n-gw topologies the `instance-mtu` config
+            # is a NOOP on neutron-openvswitch units, but that is ok.
+            'instance-mtu': self._get_instance_mtu_from_global_physnet_mtu()
+        }
+        apps = ('neutron-gateway', 'neutron-openvswitch')
+        self._configure_apps(apps, cfg)
+        cfg_ovs = {
+            # To be able to successfully clean up after the Neutron agents we
+            # need to use the 'openvswitch' `firewall-driver`.
+            'firewall-driver': 'openvswitch',
+        }
+        self._configure_apps(('neutron-openvswitch',), cfg_ovs)
+
+    def configure_ovn_mappings(self):
+        """Copy mappings from n-gw or n-ovs application."""
+        dst_apps = ('ovn-dedicated-chassis', 'ovn-chassis')
+        src_apps = ('neutron-gateway', 'neutron-openvswitch')
+        ovn_cfg = {}
+        for app in src_apps:
+            try:
+                app_cfg = zaza.model.get_application_config(app)
+                ovn_cfg['bridge-interface-mappings'] = app_cfg[
+                    'data-port']['value']
+                ovn_cfg['ovn-bridge-mappings'] = app_cfg[
+                    'bridge-mappings']['value']
+                # Use values from neutron-gateway when present, otherwise use
+                # values from neutron-openvswitch
+                break
+            except KeyError:
+                pass
+        else:
+            raise RuntimeError(
+                'None of the expected apps ({}) are present in the model.'
+                .format(src_apps)
+            )
+
+        self._configure_apps(
+            dst_apps, ovn_cfg, first_match_raise_if_none_found=True)
+
+
+def pre_migration_configuration():
+    """Perform pre-migration configuration steps.
+
+    NOTE: Doing the configuration post-deploy and after doing initial network
+    configuration is an important part of the test as we need to prove that our
+    end users would be successful in doing this in the wild.
+    """
+    # we use a helper class to leverage common setup code and the
+    # `config_change` helper
+    helper = _OVNSetupHelper()
+    helper.setUpClass()
+    # Configure `firewall-driver` and `instance-mtu` on n-gw and n-ovs units.
+    helper.configure_ngw_novs()
+    # Copy mappings from n-gw or n-ovs application to ovn-dedicated-chassis or
+    # ovn-chassis.
+    helper.configure_ovn_mappings()

zaza/openstack/charm_tests/ovn/tests.py

@@ -16,7 +16,15 @@
 
 import logging
 
+import juju
+
+import tenacity
+
+import zaza
+
+import zaza.model
 import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.openstack.utilities.generic as generic_utils
 import zaza.openstack.utilities.openstack as openstack_utils
 
 
@@ -31,10 +39,19 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest):
         """Run class setup for OVN charm operation tests."""
         super(BaseCharmOperationTest, cls).setUpClass()
         cls.services = ['NotImplemented']  # This must be overridden
+        cls.nrpe_checks = ['NotImplemented']  # This must be overridden
         cls.current_release = openstack_utils.get_os_release(
             openstack_utils.get_current_os_release_pair(
                 cls.release_application or cls.application_name))
 
+    @tenacity.retry(
+        retry=tenacity.retry_if_result(lambda ret: ret is not None),
+        # sleep for 2mins to allow 1min cron job to run...
+        wait=tenacity.wait_fixed(120),
+        stop=tenacity.stop_after_attempt(2))
+    def _retry_check_commands_on_units(self, cmds, units):
+        return generic_utils.check_commands_on_units(cmds, units)
+
     def test_pause_resume(self):
         """Run pause and resume tests.
 
@@ -45,6 +62,20 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest):
             logging.info('Testing pause resume (services="{}")'
                          .format(self.services))
 
+    def test_nrpe_configured(self):
+        """Confirm that the NRPE service check files are created."""
+        units = zaza.model.get_units(self.application_name)
+        cmds = []
+        for check_name in self.nrpe_checks:
+            cmds.append(
+                'egrep -oh /usr/local.* /etc/nagios/nrpe.d/'
+                'check_{}.cfg'.format(check_name)
+            )
+        ret = self._retry_check_commands_on_units(cmds, units)
+        if ret:
+            logging.info(ret)
+        self.assertIsNone(ret, msg=ret)
+
 
 class CentralCharmOperationTest(BaseCharmOperationTest):
     """OVN Central Charm operation tests."""
@@ -57,6 +88,22 @@ class CentralCharmOperationTest(BaseCharmOperationTest):
             'ovn-northd',
             'ovsdb-server',
         ]
+        source = zaza.model.get_application_config(
+            cls.application_name)['source']['value']
+        logging.info(source)
+        if 'train' in source:
+            cls.nrpe_checks = [
+                'ovn-northd',
+                'ovn-nb-ovsdb',
+                'ovn-sb-ovsdb',
+            ]
+        else:
+            # Ussuri or later (distro or cloudarchive)
+            cls.nrpe_checks = [
+                'ovn-northd',
+                'ovn-ovsdb-server-sb',
+                'ovn-ovsdb-server-nb',
+            ]
 
 
 class ChassisCharmOperationTest(BaseCharmOperationTest):
@@ -71,3 +118,295 @@ class ChassisCharmOperationTest(BaseCharmOperationTest):
         cls.services = [
             'ovn-controller',
         ]
+        if cls.application_name == 'ovn-chassis':
+            principal_app_name = 'magpie'
+        else:
+            principal_app_name = cls.application_name
+        source = zaza.model.get_application_config(
+            principal_app_name)['source']['value']
+        logging.info(source)
+        if 'train' in source:
+            cls.nrpe_checks = [
+                'ovn-host',
+                'ovs-vswitchd',
+                'ovsdb-server',
+            ]
+        else:
+            # Ussuri or later (distro or cloudarchive)
+            cls.nrpe_checks = [
+                'ovn-controller',
+                'ovsdb-server',
+                'ovs-vswitchd',
+            ]
+
+
+class OVSOVNMigrationTest(test_utils.BaseCharmTest):
+    """OVS to OVN migration tests."""
+
+    def setUp(self):
+        """Perform migration steps prior to validation."""
+        super(OVSOVNMigrationTest, self).setUp()
+        # These steps are here due to them having to be executed once and in a
+        # specific order prior to running any tests. The steps should still
+        # be idempotent if at all possible as a courtesy to anyone iterating
+        # on the test code.
+        try:
+            if self.one_time_init_done:
+                logging.debug('Skipping migration steps as they have already '
+                              'run.')
+                return
+        except AttributeError:
+            logging.info('Performing migration steps.')
+
+        # as we progress through the steps our target deploy status changes
+        # store it in the class instance so the individual methods can
+        # update when appropriate.
+        self.target_deploy_status = self.test_config.get(
+            'target_deploy_status', {})
+
+        # Stop Neutron agents on hypervisors
+        self._pause_units('neutron-openvswitch')
+        try:
+            self._pause_units('neutron-gateway')
+        except KeyError:
+            logging.info(
+                'No neutron-gateway in deployment, skip pausing it.')
+
+        # Add the neutron-api-plugin-ovn subordinate which will make the
+        # `neutron-api-plugin-ovn` unit appear in the deployment.
+        #
+        # NOTE: The OVN drivers will not be activated until we change the
+        # value for the `manage-neutron-plugin-legacy-mode` config.
+        self._add_neutron_api_plugin_ovn_subordinate_relation()
+
+        # Adjust MTU on overlay networks
+        #
+        # Prior to this the end user will already have lowered the MTU on their
+        # running instances through the use of the `instance-mtu` configuration
+        # option and manual reconfiguration of instances that do not use DHCP.
+        #
+        # We update the value for the MTU on the overlay networks at this point
+        # in time because:
+        #
+        # - Agents are paused and will not actually reconfigure the networks.
+        #
+        # - Making changes to non-Geneve networks are prohibited as soon as the
+        #   OVN drivers are activated.
+        #
+        # - Get the correct MTU value into the OVN database on first sync.
+        #
+        #   - This will be particularly important for any instances using
+        #     stateless IPv6 autoconfiguration (SLAAC) as there is currently
+        #     no config knob to feed MTU information into the legacy ML2+OVS
+        #     `radvd` configuration or the native OVN RA.
+        #
+        #   - Said instances will reconfigure their IPv6 MTU as soon as they
+        #     receive an RA with correct MTU when OVN takes over control.
+        self._run_migrate_mtu_action()
+
+        # Flip `manage-neutron-plugin-legacy-mode` to enable it
+        #
+        # NOTE(fnordahl): until we sync/repair the OVN DB this will make the
+        # `neutron-server` log errors. However we need the neutron unit to be
+        # unpaused while doing this to have the configuration rendered. The
+        # configuration is consumed by the `neutron-ovn-db-sync` tool.
+        self._configure_neutron_api()
+
+        # Stop the Neutron server prior to OVN DB sync/repair
+        self._pause_units('neutron-api')
+
+        # Sync the OVN DB
+        self._run_migrate_ovn_db_action()
+        # Perform the optional morphing of Neutron DB action
+        self._run_offline_neutron_morph_db_action()
+        self._resume_units('neutron-api')
+
+        # Run `cleanup` action on neutron-openvswitch units/hypervisors
+        self._run_cleanup_action('neutron-openvswitch')
+        # Run `cleanup` action on neutron-gateway units when present
+        try:
+            self._run_cleanup_action('neutron-gateway')
+        except KeyError:
+            logging.info(
+                'No neutron-gateway in deployment, skip cleanup of it.')
+
+        # Start the OVN controller on hypervisors
+        #
+        # NOTE(fnordahl): it is very important to have run cleanup prior to
+        # starting these, if you don't do that it is almost guaranteed that
+        # you will program the network to a state of infinite loop.
+        self._resume_units('ovn-chassis')
+
+        try:
+            self._resume_units('ovn-dedicated-chassis')
+        except KeyError:
+            logging.info(
+                'No ovn-dedicated-chassis in deployment, skip resume.')
+
+        # And we should be off to the races
+
+        self.one_time_init_done = True
+
+    def _add_neutron_api_plugin_ovn_subordinate_relation(self):
+        """Add relation between neutron-api and neutron-api-plugin-ovn."""
+        try:
+            logging.info('Adding relation neutron-api-plugin-ovn '
+                         '-> neutron-api')
+            zaza.model.add_relation(
+                'neutron-api-plugin-ovn', 'neutron-plugin',
+                'neutron-api:neutron-plugin-api-subordinate')
+            zaza.model.wait_for_agent_status()
+            zaza.model.wait_for_application_states(
+                states=self.test_config.get('target_deploy_status', {}))
+        except juju.errors.JujuAPIError:
+            # we were not able to add the relation, let's make sure it's
+            # because it's already there
+            assert (zaza.model.get_relation_id(
+                'neutron-api-plugin-ovn', 'neutron-api',
+                remote_interface_name='neutron-plugin-api-subordinate')
+                is not None), 'Unable to add relation required for test'
+            logging.info('--> On the other hand, did not need to add the '
+                         'relation as it was already there.')
+
+    def _configure_neutron_api(self):
+        """Set configuration option `manage-neutron-plugin-legacy-mode`."""
+        logging.info('Configuring `manage-neutron-plugin-legacy-mode` for '
+                     'neutron-api...')
+        n_api_config = {
+            'manage-neutron-plugin-legacy-mode': False,
+        }
+        with self.config_change(
+                n_api_config, n_api_config, 'neutron-api'):
+            logging.info('done')
+
+    def _run_offline_neutron_morph_db_action(self):
+        """Run offline-neutron-morph-db action."""
+        logging.info('Running the optional `offline-neutron-morph-db` action '
+                     'on neutron-api-plugin-ovn/leader')
+        generic_utils.assertActionRanOK(
+            zaza.model.run_action_on_leader(
+                'neutron-api-plugin-ovn',
+                'offline-neutron-morph-db',
+                action_params={
+                    'i-really-mean-it': True},
+                raise_on_failure=True,
+            )
+        )
+
+    def _run_migrate_ovn_db_action(self):
+        """Run migrate-ovn-db action."""
+        logging.info('Running `migrate-ovn-db` action on '
+                     'neutron-api-plugin-ovn/leader')
+        generic_utils.assertActionRanOK(
+            zaza.model.run_action_on_leader(
+                'neutron-api-plugin-ovn',
+                'migrate-ovn-db',
+                action_params={
+                    'i-really-mean-it': True},
+                raise_on_failure=True,
+            )
+        )
+
+    # Charm readiness is no guarantee for API being ready to serve requests.
+    # https://bugs.launchpad.net/charm-neutron-api/+bug/1854518
+    @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60),
+                    reraise=True, stop=tenacity.stop_after_attempt(3))
+    def _run_migrate_mtu_action(self):
+        """Run migrate-mtu action with retry.
+
+        The action is idempotent.
+
+        Due to LP: #1854518 and the point in time of the test life cycle we run
+        this action the probability for the Neutron API not being available
+        for the script to do its job is high, thus we retry.
+        """
+        logging.info('Running `migrate-mtu` action on '
+                     'neutron-api-plugin-ovn/leader')
+        generic_utils.assertActionRanOK(
+            zaza.model.run_action_on_leader(
+                'neutron-api-plugin-ovn',
+                'migrate-mtu',
+                action_params={
+                    'i-really-mean-it': True},
+                raise_on_failure=True,
+            )
+        )
+
+    def _pause_units(self, application):
+        """Pause units of application.
+
+        :param application: Name of application
+        :type application: str
+        """
+        logging.info('Pausing {} units'.format(application))
+        zaza.model.run_action_on_units(
+            [unit.entity_id
+                for unit in zaza.model.get_units(application)],
+            'pause',
+            raise_on_failure=True,
+        )
+        self.target_deploy_status.update(
+            {
+                application: {
+                    'workload-status': 'maintenance',
+                    'workload-status-message': 'Paused',
+                },
+            },
+        )
+
+    def _run_cleanup_action(self, application):
+        """Run cleanup action on application units.
+
+        :param application: Name of application
+        :type application: str
+        """
+        logging.info('Running `cleanup` action on {} units.'
+                     .format(application))
+        zaza.model.run_action_on_units(
+            [unit.entity_id
+                for unit in zaza.model.get_units(application)],
+            'cleanup',
+            action_params={
+                'i-really-mean-it': True},
+            raise_on_failure=True,
+        )
+
+    def _resume_units(self, application):
+        """Resume units of application.
+
+        :param application: Name of application
+        :type application: str
+        """
+        logging.info('Resuming {} units'.format(application))
+        zaza.model.run_action_on_units(
+            [unit.entity_id
+                for unit in zaza.model.get_units(application)],
+            'resume',
+            raise_on_failure=True,
+        )
+        self.target_deploy_status.pop(application)
+
+    def test_ovs_ovn_migration(self):
+        """Test migration of existing Neutron ML2+OVS deployment to OVN.
+
+        The test should be run after deployment and validation of a legacy
+        deployment combined with subsequent run of a network connectivity test
+        on instances created prior to the migration.
+        """
+        # The setUp method of this test class will perform the migration steps.
+        # The tests.yaml is programmed to do further validation after the
+        # migration.
+
+        # Reset the n-gw and n-ovs instance-mtu configuration option so it does
+        # not influence how further tests are executed.
+        reset_config_keys = ['instance-mtu']
+        for app in ('neutron-gateway', 'neutron-openvswitch'):
+            try:
+                zaza.model.reset_application_config(app, reset_config_keys)
+                logging.info('Reset configuration to default on "{}" for "{}"'
+                             .format(app, reset_config_keys))
+            except KeyError:
+                pass
+        zaza.model.wait_for_agent_status()
+        zaza.model.wait_for_application_states(
+            states=self.target_deploy_status)

zaza/openstack/charm_tests/policyd/tests.py

@@ -244,8 +244,8 @@ class BasePolicydSpecialization(PolicydTest,
     by the policy override in the `_rule` class variable.  The method should
     pass cleanly without the override in place.
 
-    The test_003_test_overide_is_observed will then apply the override and then
-    call get_client_and_attempt_operation() again, and this time it should
+    The test_003_test_override_is_observed will then apply the override and
+    then call get_client_and_attempt_operation() again, and this time it should
     detect the failure and raise the PolicydOperationFailedException()
     exception.  This will be detected as the override working and thus the test
     will pass.
@@ -399,7 +399,7 @@ class BasePolicydSpecialization(PolicydTest,
         return openstack_utils.get_keystone_session(
             openstack_utils.get_overcloud_auth(address=ip))
 
-    def test_003_test_overide_is_observed(self):
+    def test_003_test_override_is_observed(self):
         """Test that the override is observed by the underlying service."""
         if (openstack_utils.get_os_release() <
                 openstack_utils.get_os_release('groovy_victoria')):

zaza/openstack/charm_tests/rabbitmq_server/tests.py

@@ -276,15 +276,23 @@ class RmqTests(test_utils.OpenStackBaseTest):
         """The services can be paused and resumed."""
         logging.info('Checking pause and resume actions...')
 
+        logging.info('Waiting for the cluster to be ready')
+        rmq_utils.wait_for_cluster()
         unit = zaza.model.get_units(self.application_name)[0]
         assert unit.workload_status == "active"
 
+        logging.info('Pausing unit {}'.format(unit))
         zaza.model.run_action(unit.entity_id, "pause")
+        logging.info('Waiting until unit {} reaches "maintenance" state'
+                     ''.format(unit))
         zaza.model.block_until_unit_wl_status(unit.entity_id, "maintenance")
         unit = zaza.model.get_unit_from_name(unit.entity_id)
         assert unit.workload_status == "maintenance"
 
+        logging.info('Resuming unit {}'.format(unit))
         zaza.model.run_action(unit.entity_id, "resume")
+        logging.info('Waiting until unit {} reaches "active" state'
+                     ''.format(unit))
         zaza.model.block_until_unit_wl_status(unit.entity_id, "active")
         unit = zaza.model.get_unit_from_name(unit.entity_id)
         assert unit.workload_status == "active"
@@ -356,7 +364,7 @@ class RmqTests(test_utils.OpenStackBaseTest):
         return rmq_utils.check_unit_cluster_nodes(u, unit_node_names)
 
     @unittest.skip(
-        "Skipping as a significant rework is required, see"
+        "Skipping as a significant rework is required, see "
         "https://github.com/openstack-charmers/zaza-openstack-tests/issues/290"
     )
     def test_921_remove_and_add_unit(self):

zaza/openstack/charm_tests/security/tests.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2018 Canonical Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Encapsulate general security testing."""
-
-import unittest
-
-import zaza.model as model
-import zaza.charm_lifecycle.utils as utils
-from zaza.openstack.utilities.file_assertions import (
-    assert_path_glob,
-    assert_single_file,
-)
-
-
-def _make_test_function(application, file_details, paths=None):
-    """Generate a test function given the specified inputs.
-
-    :param application: Application name to assert file ownership on
-    :type application: str
-    :param file_details: Dictionary of file details to test
-    :type file_details: dict
-    :param paths: List of paths to test in this application
-    :type paths: Optional[list(str)]
-    :returns: Test function
-    :rtype: unittest.TestCase
-    """
-    def test(self):
-        for unit in model.get_units(application):
-            unit = unit.entity_id
-            if '*' in file_details['path']:
-                assert_path_glob(self, unit, file_details, paths)
-            else:
-                assert_single_file(self, unit, file_details)
-    return test
-
-
-def _add_tests():
-    """Add tests to the unittest.TestCase."""
-    def class_decorator(cls):
-        """Add tests based on input yaml to `cls`."""
-        files = utils.get_charm_config('./file-assertions.yaml')
-        deployed_applications = model.sync_deployed()
-        for name, attributes in files.items():
-            # Lets make sure to only add tests for deployed applications
-            if name in deployed_applications:
-                paths = [
-                    file['path'] for
-                    file in attributes['files']
-                    if "*" not in file["path"]
-                ]
-                for file in attributes['files']:
-                    test_func = _make_test_function(name, file, paths=paths)
-                    setattr(
-                        cls,
-                        'test_{}_{}'.format(name, file['path']),
-                        test_func)
-        return cls
-    return class_decorator
-
-
-class FileOwnershipTest(unittest.TestCase):
-    """Encapsulate File ownership tests."""
-
-    pass
-
-
-FileOwnershipTest = _add_tests()(FileOwnershipTest)

zaza/openstack/charm_tests/series_upgrade/tests.py

@@ -81,7 +81,7 @@ class SeriesUpgradeTest(unittest.TestCase):
                 logging.info(
                     "Running complete-cluster-series-upgrade action on leader")
                 model.run_action_on_leader(
-                    charm_name,
+                    application,
                     'complete-cluster-series-upgrade',
                     action_params={})
                 model.block_until_all_units_idle()
@@ -90,7 +90,7 @@ class SeriesUpgradeTest(unittest.TestCase):
                 logging.info(
                     "Running complete-cluster-series-upgrade action on leader")
                 model.run_action_on_leader(
-                    charm_name,
+                    application,
                     'complete-cluster-series-upgrade',
                     action_params={})
                 model.block_until_all_units_idle()

zaza/openstack/charm_tests/tempest/setup.py

@@ -26,15 +26,17 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup
 
 SETUP_ENV_VARS = {
     'neutron': ['TEST_GATEWAY', 'TEST_CIDR_EXT', 'TEST_FIP_RANGE',
-                'TEST_NAMESERVER', 'TEST_CIDR_PRIV'],
+                'TEST_NAME_SERVER', 'TEST_CIDR_PRIV'],
     'swift': ['TEST_SWIFT_IP'],
 }
 
+IGNORABLE_VARS = ['TEST_CIDR_PRIV']
+
 TEMPEST_FLAVOR_NAME = 'm1.tempest'
 TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest'
 TEMPEST_SVC_LIST = ['ceilometer', 'cinder', 'glance', 'heat', 'horizon',
-                    'ironic', 'neutron', 'nova', 'sahara', 'swift', 'trove',
-                    'zaqar']
+                    'ironic', 'neutron', 'nova', 'octavia', 'sahara', 'swift',
+                    'trove', 'zaqar']
 
 
 def add_application_ips(ctxt):
@@ -190,6 +192,7 @@ def add_environment_var_config(ctxt, services):
     :rtype: None
     """
     deploy_env = deployment_env.get_deployment_context()
+    missing_vars = []
     for svc, env_vars in SETUP_ENV_VARS.items():
         if svc in services:
             for var in env_vars:
@@ -197,9 +200,12 @@ def add_environment_var_config(ctxt, services):
                 if value:
                     ctxt[var.lower()] = value
                 else:
-                    raise ValueError(
-                        ("Environment variables {} must all be set to run this"
-                         " test").format(', '.join(env_vars)))
+                    if var not in IGNORABLE_VARS:
+                        missing_vars.append(var)
+    if missing_vars:
+        raise ValueError(
+            ("Environment variables [{}] must all be set to run this"
+             " test").format(', '.join(missing_vars)))
 
 
 def add_auth_config(ctxt):

zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2

@@ -52,9 +52,11 @@ http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-u
 
 {% if 'neutron' in enabled_services %}
 [network]
+{% if test_cidr_priv %}
 project_network_cidr = {{ test_cidr_priv }}
+{% endif %}
 public_network_id = {{ ext_net }}
-dns_servers = {{ test_nameserver }}
+dns_servers = {{ test_name_server }}
 project_networks_reachable = false
 
 [network-feature-enabled]

zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2

@@ -54,9 +54,11 @@ http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-u
 
 {% if 'neutron' in enabled_services %}
 [network]
+{% if test_cidr_priv %}
 project_network_cidr = {{ test_cidr_priv }}
+{% endif %}
 public_network_id = {{ ext_net }}
-dns_servers = {{ test_nameserver }}
+dns_servers = {{ test_name_server }}
 project_networks_reachable = false
 floating_network_name = {{ ext_net }}
 

zaza/openstack/charm_tests/test_utils.py

@@ -15,6 +15,7 @@
 import contextlib
 import logging
 import subprocess
+import sys
 import tenacity
 import unittest
 
@@ -126,15 +127,73 @@ class BaseCharmTest(unittest.TestCase):
         else:
             cls.model_name = model.get_juju_model()
         cls.test_config = lifecycle_utils.get_charm_config(fatal=False)
+
         if application_name:
             cls.application_name = application_name
         else:
-            cls.application_name = cls.test_config['charm_name']
+            charm_under_test_name = cls.test_config['charm_name']
+            deployed_app_names = model.sync_deployed(model_name=cls.model_name)
+            if charm_under_test_name in deployed_app_names:
+                # There is an application named like the charm under test.
+                # Let's consider it the application under test:
+                cls.application_name = charm_under_test_name
+            else:
+                # Let's search for any application whose name starts with the
+                # name of the charm under test and assume it's the application
+                # under test:
+                for app_name in deployed_app_names:
+                    if app_name.startswith(charm_under_test_name):
+                        cls.application_name = app_name
+                        break
+                else:
+                    logging.warning('Could not find application under test')
+                    return
+
         cls.lead_unit = model.get_lead_unit_name(
             cls.application_name,
             model_name=cls.model_name)
         logging.debug('Leader unit is {}'.format(cls.lead_unit))
 
+    def config_current_separate_non_string_type_keys(
+            self, non_string_type_keys, config_keys=None,
+            application_name=None):
+        """Obtain current config and the non-string type config separately.
+
+        If the charm config option is not string, it will not accept being
+        reverted back in "config_change()" method if the current value is None.
+        Therefore, obtain the current config and separate those out, so they
+        can be used for a separate invocation of "config_change()" with
+        reset_to_charm_default set to True.
+
+        :param config_keys: iterable of strs to index into the current config.
+                            If None, return all keys from the config
+        :type config_keys:  Optional[Iterable[str]]
+        :param non_string_type_keys: list of non-string type keys to be
+                                     separated out only if their current value
+                                     is None
+        :type non_string_type_keys: list
+        :param application_name: String application name for use when called
+                                 by a charm under test other than the object's
+                                 application.
+        :type application_name:  Optional[str]
+        :return: Dictionary of current charm configs without the
+                 non-string type keys provided, and dictionary of the
+                 non-string keys found in the supplied config_keys list.
+        :rtype: Dict[str, Any], Dict[str, None]
+        """
+        current_config = self.config_current(application_name, config_keys)
+        non_string_type_config = {}
+        if config_keys is None:
+            config_keys = list(current_config.keys())
+        for key in config_keys:
+            # We only care if the current value is None, otherwise it will
+            # not face issues being reverted by "config_change()"
+            if key in non_string_type_keys and current_config[key] is None:
+                non_string_type_config[key] = None
+                current_config.pop(key)
+
+        return current_config, non_string_type_config
+
     def config_current(self, application_name=None, keys=None):
         """Get Current Config of an application normalized into key-values.
 
@@ -181,7 +240,7 @@ class BaseCharmTest(unittest.TestCase):
 
     @contextlib.contextmanager
     def config_change(self, default_config, alternate_config,
-                      application_name=None):
+                      application_name=None, reset_to_charm_default=False):
         """Run change config tests.
 
         Change config to `alternate_config`, wait for idle workload status,
@@ -201,6 +260,12 @@ class BaseCharmTest(unittest.TestCase):
                                  by a charm under test other than the object's
                                  application.
         :type application_name: str
+        :param reset_to_charm_default: When True we will ask Juju to reset each
+                                       configuration option mentioned in the
+                                       `alternate_config` dictionary back to
+                                       the charm default and ignore the
+                                       `default_config` dictionary.
+        :type reset_to_charm_default: bool
         """
         if not application_name:
             application_name = self.application_name
@@ -245,12 +310,28 @@ class BaseCharmTest(unittest.TestCase):
 
             yield
 
-        logging.debug('Restoring charm setting to {}'.format(default_config))
-        model.set_application_config(
-            application_name,
-            self._stringed_value_config(default_config),
-            model_name=self.model_name)
+        if reset_to_charm_default:
+            logging.debug('Resetting these charm configuration options to the '
+                          'charm default: "{}"'
+                          .format(alternate_config.keys()))
+            model.reset_application_config(application_name,
+                                           list(alternate_config.keys()),
+                                           model_name=self.model_name)
+        elif default_config == alternate_config:
+            logging.debug('default_config == alternate_config, not attempting '
+                          ' to restore configuration')
+            return
+        else:
+            logging.debug('Restoring charm setting to {}'
+                          .format(default_config))
+            model.set_application_config(
+                application_name,
+                self._stringed_value_config(default_config),
+                model_name=self.model_name)
 
+        logging.debug(
+            'Waiting for units to execute config-changed hook')
+        model.wait_for_agent_status(model_name=self.model_name)
         logging.debug(
             'Waiting for units to reach target states')
         model.wait_for_application_states(
@@ -425,6 +506,50 @@ class BaseCharmTest(unittest.TestCase):
             model_name=self.model_name,
             pgrep_full=pgrep_full)
 
+    def get_my_tests_options(self, key, default=None):
+        """Retrieve tests_options for specific test.
+
+        Prefix for key is built from dot-notated absolute path to calling
+        method or function.
+
+        Example:
+           # In tests.yaml:
+           tests_options:
+             zaza.charm_tests.noop.tests.NoopTest.test_foo.key: true
+           # called from zaza.charm_tests.noop.tests.NoopTest.test_foo()
+           >>> get_my_tests_options('key')
+           True
+
+        :param key: Suffix for tests_options key.
+        :type key: str
+        :param default: Default value to return if key is not found.
+        :type default: any
+        :returns: Value associated with key in tests_options.
+        :rtype: any
+        """
+        # note that we need to do this in-line otherwise we would get the path
+        # to ourself. I guess we could create a common method that would go two
+        # frames back, but that would be kind of useless for anyone else than
+        # this method.
+        caller_path = []
+
+        # get path to module
+        caller_path.append(sys.modules[
+            sys._getframe().f_back.f_globals['__name__']].__name__)
+
+        # attempt to get class name
+        try:
+            caller_path.append(
+                sys._getframe().f_back.f_locals['self'].__class__.__name__)
+        except KeyError:
+            pass
+
+        # get method or function name
+        caller_path.append(sys._getframe().f_back.f_code.co_name)
+
+        return self.test_config.get('tests_options', {}).get(
+            '.'.join(caller_path + [key]), default)
+
 
 class OpenStackBaseTest(BaseCharmTest):
     """Generic helpers for testing OpenStack API charms."""
@@ -450,11 +575,17 @@ class OpenStackBaseTest(BaseCharmTest):
                         self.nova_client.servers,
                         server.id,
                         msg="server")
+        except AssertionError as e:
+            # Resource failed to be removed within the expected time frame,
+            # log this fact and carry on.
+            logging.warning('Gave up waiting for resource cleanup: "{}"'
+                            .format(str(e)))
         except AttributeError:
             # Test did not define self.RESOURCE_PREFIX, ignore.
             pass
 
-    def launch_guest(self, guest_name, userdata=None):
+    def launch_guest(self, guest_name, userdata=None, use_boot_volume=False,
+                     instance_key=None):
         """Launch two guests to use in tests.
 
         Note that it is up to the caller to have set the RESOURCE_PREFIX class
@@ -467,25 +598,38 @@ class OpenStackBaseTest(BaseCharmTest):
         :type guest_name: str
         :param userdata: Userdata to attach to instance
         :type userdata: Optional[str]
+        :param use_boot_volume: Whether to boot guest from a shared volume.
+        :type use_boot_volume: boolean
+        :param instance_key: Key to collect associated config data with.
+        :type instance_key: Optional[str]
         :returns: Nova instance objects
         :rtype: Server
         """
+        instance_key = instance_key or glance_setup.LTS_IMAGE_NAME
         instance_name = '{}-{}'.format(self.RESOURCE_PREFIX, guest_name)
 
-        instance = self.retrieve_guest(instance_name)
-        if instance:
-            logging.info('Removing already existing instance ({}) with '
-                         'requested name ({})'
-                         .format(instance.id, instance_name))
-            openstack_utils.delete_resource(
-                self.nova_client.servers,
-                instance.id,
-                msg="server")
+        for attempt in tenacity.Retrying(
+                stop=tenacity.stop_after_attempt(3),
+                wait=tenacity.wait_exponential(
+                    multiplier=1, min=2, max=10)):
+            with attempt:
+                old_instance_with_same_name = self.retrieve_guest(
+                    instance_name)
+                if old_instance_with_same_name:
+                    logging.info(
+                        'Removing already existing instance ({}) with '
+                        'requested name ({})'
+                        .format(old_instance_with_same_name.id, instance_name))
+                    openstack_utils.delete_resource(
+                        self.nova_client.servers,
+                        old_instance_with_same_name.id,
+                        msg="server")
 
-        return configure_guest.launch_instance(
-            glance_setup.LTS_IMAGE_NAME,
-            vm_name=instance_name,
-            userdata=userdata)
+                return configure_guest.launch_instance(
+                    instance_key,
+                    vm_name=instance_name,
+                    use_boot_volume=use_boot_volume,
+                    userdata=userdata)
 
     def launch_guests(self, userdata=None):
         """Launch two guests to use in tests.
@@ -500,15 +644,10 @@ class OpenStackBaseTest(BaseCharmTest):
         """
         launched_instances = []
         for guest_number in range(1, 2+1):
-            for attempt in tenacity.Retrying(
-                    stop=tenacity.stop_after_attempt(3),
-                    wait=tenacity.wait_exponential(
-                        multiplier=1, min=2, max=10)):
-                with attempt:
-                    launched_instances.append(
-                        self.launch_guest(
-                            guest_name='ins-{}'.format(guest_number),
-                            userdata=userdata))
+            launched_instances.append(
+                self.launch_guest(
+                    guest_name='ins-{}'.format(guest_number),
+                    userdata=userdata))
         return launched_instances
 
     def retrieve_guest(self, guest_name):

zaza/openstack/charm_tests/trilio/tests.py

@@ -25,6 +25,7 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup
 import zaza.openstack.charm_tests.test_utils as test_utils
 import zaza.openstack.configure.guest as guest_utils
 import zaza.openstack.utilities.openstack as openstack_utils
+import zaza.openstack.utilities.generic as generic_utils
 from zaza.utilities import juju as juju_utils
 
 
@@ -424,7 +425,22 @@ class TrilioBaseTest(test_utils.OpenStackBaseTest):
         workloadmgrcli.oneclick_restore(snapshot_id)
 
 
-class TrilioWLMTest(TrilioBaseTest):
+class TrilioGhostNFSShareTest(TrilioBaseTest):
+    """Tests for Trilio charms providing the ghost-share action."""
+
+    def test_ghost_nfs_share(self):
+        """Ensure ghost-share action bind mounts NFS share."""
+        generic_utils.assertActionRanOK(zaza_model.run_action(
+            self.lead_unit,
+            'ghost-share',
+            action_params={
+                'nfs-shares': '10.20.0.1:/srv/testing'
+            },
+            model_name=self.model_name)
+        )
+
+
+class TrilioWLMTest(TrilioGhostNFSShareTest):
     """Tests for Trilio Workload Manager charm."""
 
     conf_file = "/etc/workloadmgr/workloadmgr.conf"
@@ -447,7 +463,7 @@ class TrilioDMAPITest(TrilioBaseTest):
     services = ["dmapi-api"]
 
 
-class TrilioDataMoverTest(TrilioBaseTest):
+class TrilioDataMoverTest(TrilioGhostNFSShareTest):
     """Tests for Trilio Data Mover charm."""
 
     conf_file = "/etc/tvault-contego/tvault-contego.conf"

zaza/openstack/configure/network.py

@@ -184,7 +184,8 @@ def setup_sdn(network_config, keystone_session=None):
 
 
 def setup_gateway_ext_port(network_config, keystone_session=None,
-                           limit_gws=None):
+                           limit_gws=None,
+                           use_juju_wait=True):
     """Perform setup external port on Neutron Gateway.
 
     For OpenStack on OpenStack scenarios.
@@ -195,6 +196,8 @@ def setup_gateway_ext_port(network_config, keystone_session=None,
     :type keystone_session: keystoneauth1.session.Session object
     :param limit_gws: Limit the number of gateways that get a port attached
     :type limit_gws: Optional[int]
+    :param use_juju_wait: Use juju wait (default True) for model to settle
+    :type use_juju_wait: boolean
     :returns: None
     :rtype: None
     """
@@ -230,7 +233,8 @@ def setup_gateway_ext_port(network_config, keystone_session=None,
         neutron_client,
         net_id=net_id,
         add_dataport_to_netplan=add_dataport_to_netplan,
-        limit_gws=limit_gws)
+        limit_gws=limit_gws,
+        use_juju_wait=use_juju_wait)
 
 
 def run_from_cli(**kwargs):
@@ -269,6 +273,11 @@ def run_from_cli(**kwargs):
                         default="network.yaml")
     parser.add_argument("--cacert", help="Path to CA certificate bundle file",
                         default=None)
+    parser.add_argument("--no-use-juju-wait",
+                        help=("don't use juju wait for the model to settle "
+                              "(default true)"),
+                        action="store_false",
+                        default=True)
     # Handle CLI options
     options = parser.parse_args()
     net_topology = (kwargs.get('net_toplogoy') or
@@ -289,7 +298,9 @@ def run_from_cli(**kwargs):
         undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session(
             verify=cacert)
         setup_gateway_ext_port(network_config,
-                               keystone_session=undercloud_ks_sess)
+                               keystone_session=undercloud_ks_sess,
+                               use_juju_wait=cli_utils.parse_arg(
+                                   options, 'no_use_juju_wait'))
 
     overcloud_ks_sess = openstack_utils.get_overcloud_keystone_session(
         verify=cacert)

zaza/openstack/utilities/ceph.py

@@ -2,8 +2,10 @@
 import json
 import logging
 
-import zaza.openstack.utilities.openstack as openstack_utils
 import zaza.model as zaza_model
+import zaza.utilities.juju as zaza_juju
+
+import zaza.openstack.utilities.openstack as openstack_utils
 
 REPLICATED_POOL_TYPE = 'replicated'
 ERASURE_POOL_TYPE = 'erasure-coded'
@@ -204,3 +206,37 @@ def get_rbd_hash(unit_name, pool, image, model_name=None):
     if result.get('Code') != '0':
         raise zaza_model.CommandRunFailed(cmd, result)
     return result.get('Stdout').rstrip()
+
+
+def get_pools_from_broker_req(application_or_unit, model_name=None):
+    """Get pools requested by application or unit.
+
+    By retrieving and parsing broker request from relation data we can get a
+    list of pools a unit has requested.
+
+    :param application_or_unit: Name of application or unit that is at the
+                                other end of a ceph-mon relation.
+    :type application_or_unit: str
+    :param model_name: Name of Juju model to operate on
+    :type model_name: Optional[str]
+    :returns: List of pools requested.
+    :rtype: List[str]
+    :raises: KeyError
+    """
+    # NOTE: we do not pass on a name for the remote_interface_name as that
+    # varies between the Ceph consuming applications.
+    relation_data = zaza_juju.get_relation_from_unit(
+        'ceph-mon', application_or_unit, None, model_name=model_name)
+
+    # NOTE: we probably should consume the Ceph broker code from c-h but c-h is
+    # such a beast of a dependency so let's defer adding it to Zaza if we can.
+    broker_req = json.loads(relation_data['broker_req'])
+
+    # A charm may request modifications to an existing pool by adding multiple
+    # 'create-pool' broker requests so we need to deduplicate the list before
+    # returning it.
+    return list(set([
+        op['name']
+        for op in broker_req['ops']
+        if op['op'] == 'create-pool'
+    ]))

zaza/openstack/utilities/cli.py

@@ -16,6 +16,7 @@
 
 import logging
 import os
+import sys
 
 
 def parse_arg(options, arg, multiargs=False):
@@ -51,6 +52,6 @@ def setup_logging():
     rootLogger = logging.getLogger()
     rootLogger.setLevel('INFO')
     if not rootLogger.hasHandlers():
-        consoleHandler = logging.StreamHandler()
+        consoleHandler = logging.StreamHandler(sys.stdout)
         consoleHandler.setFormatter(logFormatter)
         rootLogger.addHandler(consoleHandler)

zaza/openstack/utilities/openstack.py

@@ -19,6 +19,7 @@ This module contains a number of functions for interacting with OpenStack.
 from .os_versions import (
     OPENSTACK_CODENAMES,
     SWIFT_CODENAMES,
+    OVN_CODENAMES,
     PACKAGE_CODENAMES,
     OPENSTACK_RELEASES_PAIRS,
 )
@@ -63,6 +64,8 @@ import tenacity
 import textwrap
 import urllib
 
+import zaza
+
 from zaza import model
 from zaza.openstack.utilities import (
     exceptions,
@@ -117,6 +120,14 @@ CHARM_TYPES = {
         'pkg': 'ovn-common',
         'origin_setting': 'source'
     },
+    'ceph-mon': {
+        'pkg': 'ceph-common',
+        'origin_setting': 'source'
+    },
+    'placement': {
+        'pkg': 'placement-common',
+        'origin_setting': 'openstack-origin'
+    },
 }
 
 # Older tests use the order the services appear in the list to imply
@@ -136,6 +147,8 @@ UPGRADE_SERVICES = [
     {'name': 'openstack-dashboard',
      'type': CHARM_TYPES['openstack-dashboard']},
     {'name': 'ovn-central', 'type': CHARM_TYPES['ovn-central']},
+    {'name': 'ceph-mon', 'type': CHARM_TYPES['ceph-mon']},
+    {'name': 'placement', 'type': CHARM_TYPES['placement']},
 ]
 
 
@@ -241,15 +254,17 @@ def get_designate_session_client(**kwargs):
                            **kwargs)
 
 
-def get_nova_session_client(session):
+def get_nova_session_client(session, version=2):
     """Return novaclient authenticated by keystone session.
 
     :param session: Keystone session object
     :type session: keystoneauth1.session.Session object
+    :param version: Version of client to request.
+    :type version: float
     :returns: Authenticated novaclient
     :rtype: novaclient.Client object
     """
-    return novaclient_client.Client(2, session=session)
+    return novaclient_client.Client(version, session=session)
 
 
 def get_neutron_session_client(session):
@@ -553,6 +568,20 @@ def dvr_enabled():
     return get_application_config_option('neutron-api', 'enable-dvr')
 
 
+def ngw_present():
+    """Check whether Neutron Gateway is present in deployment.
+
+    :returns: True when Neutron Gateway is present, False otherwise
+    :rtype: bool
+    """
+    try:
+        model.get_application('neutron-gateway')
+        return True
+    except KeyError:
+        pass
+    return False
+
+
 def ovn_present():
     """Check whether OVN is present in deployment.
 
@@ -630,15 +659,20 @@ def add_interface_to_netplan(server_name, mac_address):
     :type mac_address: string
     """
     if dvr_enabled():
-        application_name = 'neutron-openvswitch'
+        application_names = ('neutron-openvswitch',)
     elif ovn_present():
         # OVN chassis is a subordinate to nova-compute
-        application_name = 'nova-compute'
+        application_names = ('nova-compute', 'ovn-dedicated-chassis')
     else:
-        application_name = 'neutron-gateway'
+        application_names = ('neutron-gateway',)
 
-    unit_name = juju_utils.get_unit_name_from_host_name(
-        server_name, application_name)
+    for app_name in application_names:
+        unit_name = juju_utils.get_unit_name_from_host_name(
+            server_name, app_name)
+        if unit_name:
+            break
+    else:
+        raise RuntimeError('Unable to find unit to run commands on.')
     run_cmd_nic = "ip -f link -br -o addr|grep {}".format(mac_address)
     interface = model.run_on_unit(unit_name, run_cmd_nic)
     interface = interface['Stdout'].split(' ')[0]
@@ -669,19 +703,26 @@ def add_interface_to_netplan(server_name, mac_address):
                   "{}\nserver_name: {}".format(body_value, unit_name,
                                                interface, mac_address,
                                                server_name))
-    with tempfile.NamedTemporaryFile(mode="w") as netplan_file:
-        netplan_file.write(body_value)
-        netplan_file.flush()
-        model.scp_to_unit(unit_name, netplan_file.name,
-                          '/home/ubuntu/60-dataport.yaml', user="ubuntu")
-    run_cmd_mv = "sudo mv /home/ubuntu/60-dataport.yaml /etc/netplan/"
-    model.run_on_unit(unit_name, run_cmd_mv)
-    model.run_on_unit(unit_name, "sudo netplan apply")
+    for attempt in tenacity.Retrying(
+            stop=tenacity.stop_after_attempt(3),
+            wait=tenacity.wait_exponential(
+            multiplier=1, min=2, max=10)):
+        with attempt:
+            with tempfile.NamedTemporaryFile(mode="w") as netplan_file:
+                netplan_file.write(body_value)
+                netplan_file.flush()
+                model.scp_to_unit(
+                    unit_name, netplan_file.name,
+                    '/home/ubuntu/60-dataport.yaml', user="ubuntu")
+            run_cmd_mv = "sudo mv /home/ubuntu/60-dataport.yaml /etc/netplan/"
+            model.run_on_unit(unit_name, run_cmd_mv)
+            model.run_on_unit(unit_name, "sudo netplan apply")
 
 
 def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
                                add_dataport_to_netplan=False,
-                               limit_gws=None):
+                               limit_gws=None,
+                               use_juju_wait=True):
     """Configure the neturong-gateway external port.
 
     :param novaclient: Authenticated novaclient
@@ -692,6 +733,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
     :type net_id: string
     :param limit_gws: Limit the number of gateways that get a port attached
     :type limit_gws: Optional[int]
+    :param use_juju_wait: Whether to use juju wait to wait for the model to
+        settle once the gateway has been configured. Default is True
+    :type use_juju_wait: boolean
     """
     deprecated_extnet_mode = deprecated_external_networking()
 
@@ -715,6 +759,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
         except KeyError:
             # neutron-gateway not in deployment
             pass
+    elif ngw_present():
+        uuids = itertools.islice(get_gateway_uuids(), limit_gws)
+        application_names = ['neutron-gateway']
     elif ovn_present():
         uuids = itertools.islice(get_ovn_uuids(), limit_gws)
         application_names = ['ovn-chassis']
@@ -729,8 +776,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
         config.update({'ovn-bridge-mappings': 'physnet1:br-ex'})
         add_dataport_to_netplan = True
     else:
-        uuids = itertools.islice(get_gateway_uuids(), limit_gws)
-        application_names = ['neutron-gateway']
+        raise RuntimeError('Unable to determine charm network topology.')
 
     if not net_id:
         net_id = get_admin_net(neutronclient)['id']
@@ -745,8 +791,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
                     'Neutron Gateway already has additional port')
                 break
         else:
-            logging.info('Attaching additional port to instance, '
-                         'connected to net id: {}'.format(net_id))
+            logging.info('Attaching additional port to instance ("{}"), '
+                         'connected to net id: {}'
+                         .format(uuid, net_id))
             body_value = {
                 "port": {
                     "admin_state_up": True,
@@ -796,7 +843,17 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
         # NOTE(fnordahl): We are stuck with juju_wait until we figure out how
         # to deal with all the non ['active', 'idle', 'Unit is ready.']
         # workload/agent states and msgs that our mojo specs are exposed to.
-        juju_wait.wait(wait_for_workload=True)
+        if use_juju_wait:
+            juju_wait.wait(wait_for_workload=True)
+        else:
+            zaza.model.wait_for_agent_status()
+            # TODO: shouldn't access get_charm_config() here as it relies on
+            # ./tests/tests.yaml existing by default (regardless of the
+            # fatal=False) ... it's not great design.
+            test_config = zaza.charm_lifecycle.utils.get_charm_config(
+                fatal=False)
+            zaza.model.wait_for_application_states(
+                states=test_config.get('target_deploy_status', {}))
 
 
 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
@@ -1425,8 +1482,23 @@ def get_swift_codename(version):
     :returns: Codename for swift
     :rtype: string
     """
-    codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v]
-    return codenames[0]
+    return _get_special_codename(version, SWIFT_CODENAMES)
+
+
+def get_ovn_codename(version):
+    """Determine OpenStack codename that corresponds to OVN version.
+
+    :param version: Version of OVN
+    :type version: string
+    :returns: Codename for OVN
+    :rtype: string
+    """
+    return _get_special_codename(version, OVN_CODENAMES)
+
+
+def _get_special_codename(version, codenames):
+    found = [k for k, v in six.iteritems(codenames) if version in v]
+    return found[0]
 
 
 def get_os_code_info(package, pkg_version):
@@ -1439,7 +1511,6 @@ def get_os_code_info(package, pkg_version):
     :returns: Codename for package
     :rtype: string
     """
-    # {'code_num': entry, 'code_name': OPENSTACK_CODENAMES[entry]}
     # Remove epoch if it exists
     if ':' in pkg_version:
         pkg_version = pkg_version.split(':')[1:][0]
@@ -1463,6 +1534,8 @@ def get_os_code_info(package, pkg_version):
         # < Liberty co-ordinated project versions
         if 'swift' in package:
             return get_swift_codename(vers)
+        elif 'ovn' in package:
+            return get_ovn_codename(vers)
         else:
             return OPENSTACK_CODENAMES[vers]
 
@@ -1832,7 +1905,8 @@ def download_image(image_url, target_file):
 
 def _resource_reaches_status(resource, resource_id,
                              expected_status='available',
-                             msg='resource'):
+                             msg='resource',
+                             resource_attribute='status'):
     """Wait for an openstack resources status to reach an expected status.
 
        Wait for an openstack resources status to reach an expected status
@@ -1847,20 +1921,22 @@ def _resource_reaches_status(resource, resource_id,
     :param expected_status: status to expect resource to reach
     :type expected_status: str
     :param msg: text to identify purpose in logging
-    :type msy: str
+    :type msg: str
+    :param resource_attribute: Resource attribute to check against
+    :type resource_attribute: str
     :raises: AssertionError
     """
-    resource_status = resource.get(resource_id).status
-    logging.info(resource_status)
-    assert resource_status == expected_status, (
-        "Resource in {} state, waiting for {}" .format(resource_status,
-                                                       expected_status,))
+    resource_status = getattr(resource.get(resource_id), resource_attribute)
+    logging.info("{}: resource {} in {} state, waiting for {}".format(
+        msg, resource_id, resource_status, expected_status))
+    assert resource_status == expected_status
 
 
 def resource_reaches_status(resource,
                             resource_id,
                             expected_status='available',
                             msg='resource',
+                            resource_attribute='status',
                             wait_exponential_multiplier=1,
                             wait_iteration_max_time=60,
                             stop_after_attempt=8,
@@ -1880,6 +1956,8 @@ def resource_reaches_status(resource,
     :type expected_status: str
     :param msg: text to identify purpose in logging
     :type msg: str
+    :param resource_attribute: Resource attribute to check against
+    :type resource_attribute: str
     :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier
                                         seconds between each retry
     :type wait_exponential_multiplier: int
@@ -1901,7 +1979,8 @@ def resource_reaches_status(resource,
         resource,
         resource_id,
         expected_status,
-        msg)
+        msg,
+        resource_attribute)
 
 
 def _resource_removed(resource, resource_id, msg="resource"):
@@ -1916,8 +1995,8 @@ def _resource_removed(resource, resource_id, msg="resource"):
     :raises: AssertionError
     """
     matching = [r for r in resource.list() if r.id == resource_id]
-    logging.debug("Resource {} still present".format(resource_id))
-    assert len(matching) == 0, "Resource {} still present".format(resource_id)
+    logging.debug("{}: resource {} still present".format(msg, resource_id))
+    assert len(matching) == 0
 
 
 def resource_removed(resource,
@@ -2011,7 +2090,8 @@ def delete_volume_backup(cinder, vol_backup_id):
 
 
 def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
-                           visibility='public', container_format='bare'):
+                           visibility='public', container_format='bare',
+                           backend=None):
     """Upload the given image to glance and apply the given label.
 
     :param glance: Authenticated glanceclient
@@ -2037,7 +2117,7 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
         disk_format=disk_format,
         visibility=visibility,
         container_format=container_format)
-    glance.images.upload(image.id, open(local_path, 'rb'))
+    glance.images.upload(image.id, open(local_path, 'rb'), backend=backend)
 
     resource_reaches_status(
         glance.images,
@@ -2049,7 +2129,8 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
 
 
 def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
-                 properties=None):
+                 properties=None, backend=None, disk_format='qcow2',
+                 visibility='public', container_format='bare'):
     """Download the image and upload it to glance.
 
     Download an image from image_url and upload it to glance labelling
@@ -2083,7 +2164,10 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
     if not os.path.exists(local_path):
         download_image(image_url, local_path)
 
-    image = upload_image_to_glance(glance, local_path, image_name)
+    image = upload_image_to_glance(
+        glance, local_path, image_name, backend=backend,
+        disk_format=disk_format, visibility=visibility,
+        container_format=container_format)
     for tag in tags:
         result = glance.image_tags.update(image.id, tag)
         logging.debug(

zaza/openstack/utilities/os_versions.py

@@ -71,7 +71,6 @@ OPENSTACK_RELEASES_PAIRS = [
     'eoan_train', 'bionic_ussuri', 'focal_ussuri',
     'focal_victoria', 'groovy_victoria']
 
-# The ugly duckling - must list releases oldest to newest
 SWIFT_CODENAMES = OrderedDict([
     ('diablo',
         ['1.4.3']),
@@ -113,6 +112,15 @@ SWIFT_CODENAMES = OrderedDict([
         ['2.25.0']),
 ])
 
+OVN_CODENAMES = OrderedDict([
+    ('train',
+        ['2.12']),
+    ('ussuri',
+        ['20.03']),
+    ('victoria',
+        ['20.06']),
+])
+
 # >= Liberty version->codename mapping
 PACKAGE_CODENAMES = {
     'nova-common': OrderedDict([
@@ -245,8 +253,16 @@ PACKAGE_CODENAMES = {
         ('10', 'ussuri'),
         ('11', 'victoria'),
     ]),
-    'ovn-common': OrderedDict([
+    'ceph-common': OrderedDict([
+        ('10', 'mitaka'),  # jewel
+        ('12', 'queens'),  # luminous
+        ('13', 'rocky'),   # mimic
+        ('14', 'train'),   # nautilus
+        ('15', 'ussuri'),  # octopus
+    ]),
+    'placement-common': OrderedDict([
         ('2', 'train'),
-        ('20', 'ussuri'),
+        ('3', 'ussuri'),
+        ('4', 'victoria'),
     ]),
 }