Merge branch 'master' into bug/1706699

tox.ini

@@ -1,6 +1,22 @@
 [tox]
 envlist = pep8, py3
 skipsdist = True
+# NOTE: Avoid build/test env pollution by not enabling sitepackages.
+sitepackages = False
+# NOTE: Avoid false positives by not skipping missing interpreters.
+skip_missing_interpreters = False
+# NOTES:
+# * We avoid the new dependency resolver by pinning pip < 20.3, see
+#   https://github.com/pypa/pip/issues/9187
+# * Pinning dependencies requires tox >= 3.2.0, see
+#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
+# * It is also necessary to pin virtualenv as a newer virtualenv would still
+#   lead to fetching the latest pip in the func* tox targets, see
+#   https://stackoverflow.com/a/38133283
+requires = pip < 20.3
+           virtualenv < 20.0
+# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
+minversion = 3.2.0
 
 [testenv]
 setenv = VIRTUAL_ENV={envdir}

unit_tests/__init__.py

@@ -11,3 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import sys
+import unittest.mock as mock
+
+sys.modules['zaza.utilities.maas'] = mock.MagicMock()

unit_tests/utilities/test_zaza_utilities_openstack.py

@@ -17,6 +17,8 @@ import datetime
 import io
 import mock
 import subprocess
+import sys
+import unittest
 import tenacity
 
 import unit_tests.utils as ut_utils
@@ -191,6 +193,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.patch_object(openstack_utils, 'get_application_config_option')
         self.patch_object(openstack_utils, 'get_keystone_ip')
         self.patch_object(openstack_utils, "get_current_os_versions")
+        self.patch_object(openstack_utils, "get_remote_ca_cert_file")
         self.patch_object(openstack_utils.juju_utils, 'leader_get')
         if tls_relation:
             self.patch_object(openstack_utils.model, "scp_from_unit")
@@ -204,6 +207,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.get_relation_id.return_value = None
         self.get_application_config_option.return_value = None
         self.leader_get.return_value = 'openstack'
+        self.get_remote_ca_cert_file.return_value = None
         if tls_relation or ssl_cert:
             port = 35357
             transport = 'https'
@@ -245,7 +249,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
                 'API_VERSION': 3,
             }
         if tls_relation:
-            expect['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT
+            self.get_remote_ca_cert_file.return_value = '/tmp/a.cert'
+            expect['OS_CACERT'] = '/tmp/a.cert'
         self.assertEqual(openstack_utils.get_overcloud_auth(),
                          expect)
 
@@ -1262,34 +1267,218 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.get_application.side_effect = KeyError
         self.assertFalse(openstack_utils.ngw_present())
 
-    def test_configure_gateway_ext_port(self):
-        # FIXME: this is not a complete unit test for the function as one did
-        # not exist at all I'm adding this to test one bit and we'll add more
-        # as we go.
+    def test_get_charm_networking_data(self):
         self.patch_object(openstack_utils, 'deprecated_external_networking')
         self.patch_object(openstack_utils, 'dvr_enabled')
         self.patch_object(openstack_utils, 'ovn_present')
         self.patch_object(openstack_utils, 'ngw_present')
+        self.patch_object(openstack_utils, 'get_ovs_uuids')
         self.patch_object(openstack_utils, 'get_gateway_uuids')
-        self.patch_object(openstack_utils, 'get_admin_net')
+        self.patch_object(openstack_utils, 'get_ovn_uuids')
+        self.patch_object(openstack_utils.model, 'get_application')
         self.dvr_enabled.return_value = False
         self.ovn_present.return_value = False
-        self.ngw_present.return_value = True
-        self.get_admin_net.return_value = {'id': 'fakeid'}
+        self.ngw_present.return_value = False
+        self.get_ovs_uuids.return_value = []
+        self.get_gateway_uuids.return_value = []
+        self.get_ovn_uuids.return_value = []
+        self.get_application.side_effect = KeyError
 
-        novaclient = mock.MagicMock()
-        neutronclient = mock.MagicMock()
-
-        def _fake_empty_generator(empty=True):
-            if empty:
-                return
-            yield
-
-        self.get_gateway_uuids.side_effect = _fake_empty_generator
         with self.assertRaises(RuntimeError):
-            openstack_utils.configure_gateway_ext_port(
-                novaclient, neutronclient)
-        # provide a uuid and check that we don't raise RuntimeError
-        self.get_gateway_uuids.side_effect = ['fake-uuid']
-        openstack_utils.configure_gateway_ext_port(
-            novaclient, neutronclient)
+            openstack_utils.get_charm_networking_data()
+        self.ngw_present.return_value = True
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVS,
+                ['neutron-gateway'],
+                mock.ANY,
+                'data-port',
+                {}))
+        self.dvr_enabled.return_value = True
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR,
+                ['neutron-gateway', 'neutron-openvswitch'],
+                mock.ANY,
+                'data-port',
+                {}))
+        self.ngw_present.return_value = False
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT,
+                ['neutron-openvswitch'],
+                mock.ANY,
+                'data-port',
+                {}))
+        self.dvr_enabled.return_value = False
+        self.ovn_present.return_value = True
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVN,
+                ['ovn-chassis'],
+                mock.ANY,
+                'bridge-interface-mappings',
+                {'ovn-bridge-mappings': 'physnet1:br-ex'}))
+        self.get_application.side_effect = None
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVN,
+                ['ovn-chassis', 'ovn-dedicated-chassis'],
+                mock.ANY,
+                'bridge-interface-mappings',
+                {'ovn-bridge-mappings': 'physnet1:br-ex'}))
+
+    def test_get_cacert(self):
+        self.patch_object(openstack_utils.os.path, 'exists')
+        results = {
+            'tests/vault_juju_ca_cert.crt': True}
+        self.exists.side_effect = lambda x: results[x]
+        self.assertEqual(
+            openstack_utils.get_cacert(),
+            'tests/vault_juju_ca_cert.crt')
+
+        results = {
+            'tests/vault_juju_ca_cert.crt': False,
+            'tests/keystone_juju_ca_cert.crt': True}
+        self.assertEqual(
+            openstack_utils.get_cacert(),
+            'tests/keystone_juju_ca_cert.crt')
+
+        results = {
+            'tests/vault_juju_ca_cert.crt': False,
+            'tests/keystone_juju_ca_cert.crt': False}
+        self.assertIsNone(openstack_utils.get_cacert())
+
+    def test_get_remote_ca_cert_file(self):
+        self.patch_object(openstack_utils.model, 'get_first_unit_name')
+        self.patch_object(
+            openstack_utils,
+            '_get_remote_ca_cert_file_candidates')
+        self.patch_object(openstack_utils.model, 'scp_from_unit')
+        self.patch_object(openstack_utils.os.path, 'exists')
+        self.patch_object(openstack_utils.shutil, 'move')
+        self.patch_object(openstack_utils.os, 'chmod')
+        self.patch_object(openstack_utils.tempfile, 'NamedTemporaryFile')
+        enter_mock = mock.MagicMock()
+        enter_mock.__enter__.return_value.name = 'tempfilename'
+        self.NamedTemporaryFile.return_value = enter_mock
+        self.get_first_unit_name.return_value = 'neutron-api/0'
+        self._get_remote_ca_cert_file_candidates.return_value = [
+            '/tmp/ca1.cert']
+        self.exists.return_value = True
+
+        openstack_utils.get_remote_ca_cert_file('neutron-api')
+        self.scp_from_unit.assert_called_once_with(
+            'neutron-api/0',
+            '/tmp/ca1.cert',
+            'tempfilename')
+        self.chmod.assert_called_once_with('tests/ca1.cert', 0o644)
+        self.move.assert_called_once_with('tempfilename', 'tests/ca1.cert')
+
+
+class TestAsyncOpenstackUtils(ut_utils.AioTestCase):
+
+    def setUp(self):
+        super(TestAsyncOpenstackUtils, self).setUp()
+        if sys.version_info < (3, 6, 0):
+            raise unittest.SkipTest("Can't AsyncMock in py35")
+        model_mock = mock.MagicMock()
+        test_mock = mock.MagicMock()
+
+        class AsyncContextManagerMock(test_mock):
+            async def __aenter__(self):
+                return self
+
+            async def __aexit__(self, *args):
+                pass
+
+        self.model_mock = model_mock
+        self.patch_object(openstack_utils.zaza.model, "async_block_until")
+
+        async def _block_until(f, timeout):
+            # Store the result of the call to _check_ca_present to validate
+            # tests
+            self.result = await f()
+        self.async_block_until.side_effect = _block_until
+        self.patch('zaza.model.run_in_model', name='_run_in_model')
+        self._run_in_model.return_value = AsyncContextManagerMock
+        self._run_in_model().__aenter__.return_value = self.model_mock
+
+    async def test_async_block_until_ca_exists(self):
+        def _get_action_output(stdout, code, stderr=None):
+            stderr = stderr or ''
+            action = mock.MagicMock()
+            action.data = {
+                'results': {
+                    'Code': code,
+                    'Stderr': stderr,
+                    'Stdout': stdout}}
+            return action
+        results = {
+            '/tmp/missing.cert': _get_action_output(
+                '',
+                '1',
+                'cat: /tmp/missing.cert: No such file or directory'),
+            '/tmp/good.cert': _get_action_output('CERTIFICATE', '0')}
+
+        async def _run(command, timeout=None):
+            return results[command.split()[-1]]
+        self.unit1 = mock.MagicMock()
+        self.unit2 = mock.MagicMock()
+        self.unit2.run.side_effect = _run
+        self.unit1.run.side_effect = _run
+        self.units = [self.unit1, self.unit2]
+        _units = mock.MagicMock()
+        _units.units = self.units
+        self.model_mock.applications = {
+            'keystone': _units
+        }
+        self.patch_object(
+            openstack_utils,
+            "_async_get_remote_ca_cert_file_candidates")
+
+        # Test a missing cert then a good cert.
+        self._async_get_remote_ca_cert_file_candidates.return_value = [
+            '/tmp/missing.cert',
+            '/tmp/good.cert']
+        await openstack_utils.async_block_until_ca_exists(
+            'keystone',
+            'CERTIFICATE')
+        self.assertTrue(self.result)
+
+        # Test a single missing
+        self._async_get_remote_ca_cert_file_candidates.return_value = [
+            '/tmp/missing.cert']
+        await openstack_utils.async_block_until_ca_exists(
+            'keystone',
+            'CERTIFICATE')
+        self.assertFalse(self.result)
+
+    async def test__async_get_remote_ca_cert_file_candidates(self):
+        self.patch_object(openstack_utils.zaza.model, "async_get_relation_id")
+        rel_id_out = {
+        }
+
+        def _get_relation_id(app, cert_app, model_name, remote_interface_name):
+            return rel_id_out[cert_app]
+        self.async_get_relation_id.side_effect = _get_relation_id
+
+        rel_id_out['vault'] = 'certs:1'
+        r = await openstack_utils._async_get_remote_ca_cert_file_candidates(
+            'neutron-api', 'mymodel')
+        self.assertEqual(
+            r,
+            ['/usr/local/share/ca-certificates/vault_juju_ca_cert.crt',
+             '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'])
+
+        rel_id_out['vault'] = None
+        r = await openstack_utils._async_get_remote_ca_cert_file_candidates(
+            'neutron-api', 'mymodel')
+        self.assertEqual(
+            r,
+            ['/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'])

unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import asyncio
 import mock
 import sys
 import unittest
@@ -139,28 +138,7 @@ class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase):
         self.assertEqual(expected, config)
 
 
-class AioTestCase(ut_utils.BaseTestCase):
-    def __init__(self, methodName='runTest', loop=None):
-        self.loop = loop or asyncio.get_event_loop()
-        self._function_cache = {}
-        super(AioTestCase, self).__init__(methodName=methodName)
-
-    def coroutine_function_decorator(self, func):
-        def wrapper(*args, **kw):
-            return self.loop.run_until_complete(func(*args, **kw))
-        return wrapper
-
-    def __getattribute__(self, item):
-        attr = object.__getattribute__(self, item)
-        if asyncio.iscoroutinefunction(attr) and item.startswith('test_'):
-            if item not in self._function_cache:
-                self._function_cache[item] = (
-                    self.coroutine_function_decorator(attr))
-            return self._function_cache[item]
-        return attr
-
-
-class TestParallelSeriesUpgrade(AioTestCase):
+class TestParallelSeriesUpgrade(ut_utils.AioTestCase):
     def setUp(self):
         super(TestParallelSeriesUpgrade, self).setUp()
         if sys.version_info < (3, 6, 0):

unit_tests/utils.py

@@ -19,6 +19,7 @@
 
 """Module to provide helper for writing unit tests."""
 
+import asyncio
 import contextlib
 import io
 import mock
@@ -96,3 +97,24 @@ class BaseTestCase(unittest.TestCase):
             started.return_value = return_value
         self._patches_start[name] = started
         setattr(self, name, started)
+
+
+class AioTestCase(BaseTestCase):
+    def __init__(self, methodName='runTest', loop=None):
+        self.loop = loop or asyncio.get_event_loop()
+        self._function_cache = {}
+        super(AioTestCase, self).__init__(methodName=methodName)
+
+    def coroutine_function_decorator(self, func):
+        def wrapper(*args, **kw):
+            return self.loop.run_until_complete(func(*args, **kw))
+        return wrapper
+
+    def __getattribute__(self, item):
+        attr = object.__getattribute__(self, item)
+        if asyncio.iscoroutinefunction(attr) and item.startswith('test_'):
+            if item not in self._function_cache:
+                self._function_cache[item] = (
+                    self.coroutine_function_decorator(attr))
+            return self._function_cache[item]
+        return attr

zaza/openstack/charm_tests/hacluster/tests.py

@@ -125,8 +125,8 @@ class HaclusterScalebackTest(HaclusterBaseTest):
 
         logging.info('Waiting for model to settle')
         zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active')
-        # NOTE(lourot): the principle application remains blocked after scaling
-        # back up until lp:1400481 is solved.
-        zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked')
+        # NOTE(lourot): the principle application sometimes remain blocked
+        # after scaling back up until lp:1400481 is solved.
+        # zaza.model.block_until_unit_wl_status(other_principle_unit, 'active')
         zaza.model.block_until_all_units_idle()
         logging.debug('OK')

zaza/openstack/charm_tests/keystone/setup.py

@@ -41,9 +41,8 @@ def wait_for_cacert(model_name=None):
     :type model_name: str
     """
     logging.info("Waiting for cacert")
-    zaza.model.block_until_file_has_contents(
+    zaza.openstack.utilities.openstack.block_until_ca_exists(
         'keystone',
-        openstack_utils.KEYSTONE_REMOTE_CACERT,
         'CERTIFICATE',
         model_name=model_name)
     zaza.model.block_until_all_units_idle(model_name=model_name)

zaza/openstack/charm_tests/keystone/tests.py

@@ -229,7 +229,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest):
                     'OS_DOMAIN_NAME': DEMO_DOMAIN,
                 }
                 if self.tls_rid:
-                    openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT
+                    openrc['OS_CACERT'] = openstack_utils.get_cacert()
                     openrc['OS_AUTH_URL'] = (
                         openrc['OS_AUTH_URL'].replace('http', 'https'))
                 logging.info('keystone IP {}'.format(ip))
@@ -259,7 +259,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest):
         """
         def _validate_token_data(openrc):
             if self.tls_rid:
-                openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT
+                openrc['OS_CACERT'] = openstack_utils.get_cacert()
                 openrc['OS_AUTH_URL'] = (
                     openrc['OS_AUTH_URL'].replace('http', 'https'))
             logging.info('keystone IP {}'.format(ip))

zaza/openstack/charm_tests/neutron/setup.py

@@ -15,6 +15,7 @@
 """Setup for Neutron deployments."""
 
 import functools
+import logging
 
 from zaza.openstack.configure import (
     network,
@@ -89,12 +90,25 @@ def basic_overcloud_network(limit_gws=None):
         'configure_gateway_ext_port_use_juju_wait', True)
 
     # Handle network for OpenStack-on-OpenStack scenarios
-    if juju_utils.get_provider_type() == "openstack":
+    provider_type = juju_utils.get_provider_type()
+    if provider_type == "openstack":
         undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session()
         network.setup_gateway_ext_port(network_config,
                                        keystone_session=undercloud_ks_sess,
-                                       limit_gws=None,
+                                       limit_gws=limit_gws,
                                        use_juju_wait=use_juju_wait)
+    elif provider_type == "maas":
+        # NOTE(fnordahl): After validation of the MAAS+Netplan Open vSwitch
+        # integration support, we would most likely want to add multiple modes
+        # of operation with MAAS.
+        #
+        # Perform charm based OVS configuration
+        openstack_utils.configure_charmed_openstack_on_maas(
+            network_config, limit_gws=limit_gws)
+    else:
+        logging.warning('Unknown Juju provider type, "{}", will not perform'
+                        ' charm network configuration.'
+                        .format(provider_type))
 
     # Confugre the overcloud network
     network.setup_sdn(network_config, keystone_session=keystone_session)

zaza/openstack/charm_tests/nova/setup.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Code for configureing nova."""
+"""Code for configuring nova."""
+
+import tenacity
 
 import zaza.openstack.utilities.openstack as openstack_utils
 from zaza.openstack.utilities import (
@@ -21,6 +23,9 @@ from zaza.openstack.utilities import (
 import zaza.openstack.charm_tests.nova.utils as nova_utils
 
 
+@tenacity.retry(stop=tenacity.stop_after_attempt(3),
+                wait=tenacity.wait_exponential(
+                multiplier=1, min=2, max=10))
 def create_flavors(nova_client=None):
     """Create basic flavors.
 
@@ -43,6 +48,9 @@ def create_flavors(nova_client=None):
                 flavorid=nova_utils.FLAVORS[flavor]['flavorid'])
 
 
+@tenacity.retry(stop=tenacity.stop_after_attempt(3),
+                wait=tenacity.wait_exponential(
+                multiplier=1, min=2, max=10))
 def manage_ssh_key(nova_client=None):
     """Create basic flavors.
 

zaza/openstack/charm_tests/nova/tests.py

@@ -495,17 +495,24 @@ class SecurityTests(test_utils.OpenStackBaseTest):
         # Changes fixing the below expected failures will be made following
         # this initial work to get validation in. There will be bugs targeted
         # to each one and resolved independently where possible.
-
         expected_failures = [
-            'is-volume-encryption-enabled',
-            'validate-uses-tls-for-glance',
-            'validate-uses-tls-for-keystone',
         ]
         expected_passes = [
             'validate-file-ownership',
             'validate-file-permissions',
             'validate-uses-keystone',
         ]
+        tls_checks = [
+            'validate-uses-tls-for-glance',
+            'validate-uses-tls-for-keystone',
+        ]
+        if zaza.model.get_relation_id(
+                'nova-cloud-controller',
+                'vault',
+                remote_interface_name='certificates'):
+            expected_passes.extend(tls_checks)
+        else:
+            expected_failures.extend(tls_checks)
 
         for unit in zaza.model.get_units(self.application_name,
                                          model_name=self.model_name):
@@ -519,4 +526,4 @@ class SecurityTests(test_utils.OpenStackBaseTest):
                     action_params={}),
                 expected_passes,
                 expected_failures,
-                expected_to_pass=False)
+                expected_to_pass=not len(expected_failures))

zaza/openstack/charm_tests/octavia/tests.py

@@ -280,7 +280,13 @@ class LBAASv2Test(test_utils.OpenStackBaseTest):
                 lambda x: octavia_client.member_show(
                     pool_id=pool_id, member_id=x),
                 member_id,
-                operating_status='ONLINE' if monitor else '')
+                operating_status='')
+            # Temporarily disable this check until we figure out why
+            # operational_status sometimes does not become 'ONLINE'
+            # while the member does indeed work and the subsequent
+            # retrieval of payload through loadbalancer is successful
+            # ref LP: #1896729.
+            #    operating_status='ONLINE' if monitor else '')
             logging.info(resp)
         return lb
 

zaza/openstack/charm_tests/openstack_dashboard/tests.py

@@ -67,7 +67,6 @@ def _login(dashboard_url, domain, username, password, cafile=None):
     # start session, get csrftoken
     client = requests.session()
     client.get(auth_url, verify=cafile)
-
     if 'csrftoken' in client.cookies:
         csrftoken = client.cookies['csrftoken']
     else:
@@ -163,7 +162,60 @@ def _do_request(request, cafile=None):
     return urllib.request.urlopen(request, cafile=cafile)
 
 
-class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
+class OpenStackDashboardBase():
+    """Mixin for interacting with Horizon."""
+
+    def get_base_url(self):
+        """Return the base url for http(s) requests.
+
+        :returns: URL
+        :rtype: str
+        """
+        vip = (zaza_model.get_application_config(self.application_name)
+               .get("vip").get("value"))
+        if vip:
+            ip = vip
+        else:
+            unit = zaza_model.get_unit_from_name(
+                zaza_model.get_lead_unit_name(self.application_name))
+            ip = unit.public_address
+
+        logging.debug("Dashboard ip is:{}".format(ip))
+        scheme = 'http'
+        if self.use_https:
+            scheme = 'https'
+        url = '{}://{}'.format(scheme, ip)
+        return url
+
+    def get_horizon_url(self):
+        """Return the url for acccessing horizon.
+
+        :returns: Horizon URL
+        :rtype: str
+        """
+        url = '{}/horizon'.format(self.get_base_url())
+        logging.info("Horizon URL is: {}".format(url))
+        return url
+
+    @property
+    def use_https(self):
+        """Whether dashboard is using https.
+
+        :returns: Whether dashboard is using https
+        :rtype: boolean
+        """
+        use_https = False
+        vault_relation = zaza_model.get_relation_id(
+            self.application,
+            'vault',
+            remote_interface_name='certificates')
+        if vault_relation:
+            use_https = True
+        return use_https
+
+
+class OpenStackDashboardTests(test_utils.OpenStackBaseTest,
+                              OpenStackDashboardBase):
     """Encapsulate openstack dashboard charm tests."""
 
     @classmethod
@@ -171,13 +223,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
         """Run class setup for running openstack dashboard charm tests."""
         super(OpenStackDashboardTests, cls).setUpClass()
         cls.application = 'openstack-dashboard'
-        cls.use_https = False
-        vault_relation = zaza_model.get_relation_id(
-            cls.application,
-            'vault',
-            remote_interface_name='certificates')
-        if vault_relation:
-            cls.use_https = True
 
     def test_050_local_settings_permissions_regression_check_lp1755027(self):
         """Assert regression check lp1755027.
@@ -302,39 +347,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
                         mismatches.append(msg)
         return mismatches
 
-    def get_base_url(self):
-        """Return the base url for http(s) requests.
-
-        :returns: URL
-        :rtype: str
-        """
-        vip = (zaza_model.get_application_config(self.application_name)
-               .get("vip").get("value"))
-        if vip:
-            ip = vip
-        else:
-            unit = zaza_model.get_unit_from_name(
-                zaza_model.get_lead_unit_name(self.application_name))
-            ip = unit.public_address
-
-        logging.debug("Dashboard ip is:{}".format(ip))
-        scheme = 'http'
-        if self.use_https:
-            scheme = 'https'
-        url = '{}://{}'.format(scheme, ip)
-        logging.debug("Base URL is: {}".format(url))
-        return url
-
-    def get_horizon_url(self):
-        """Return the url for acccessing horizon.
-
-        :returns: Horizon URL
-        :rtype: str
-        """
-        url = '{}/horizon'.format(self.get_base_url())
-        logging.info("Horizon URL is: {}".format(url))
-        return url
-
     def test_400_connection(self):
         """Test that dashboard responds to http request.
 
@@ -450,7 +462,8 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
             logging.info("Testing pause resume")
 
 
-class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization):
+class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization,
+                                     OpenStackDashboardBase):
     """Test the policyd override using the dashboard."""
 
     good = {
@@ -476,6 +489,7 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization):
         super(OpenStackDashboardPolicydTests, cls).setUpClass(
             application_name="openstack-dashboard")
         cls.application_name = "openstack-dashboard"
+        cls.application = cls.application_name
 
     def get_client_and_attempt_operation(self, ip):
         """Attempt to list users on the openstack-dashboard service.
@@ -500,7 +514,7 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization):
         username = 'admin',
         password = overcloud_auth['OS_PASSWORD'],
         client, response = _login(
-            unit.public_address, domain, username, password)
+            self.get_horizon_url(), domain, username, password)
         # now attempt to get the domains page
         _url = self.url.format(unit.public_address)
         result = client.get(_url)

zaza/openstack/charm_tests/policyd/tests.py

@@ -337,8 +337,7 @@ class BasePolicydSpecialization(PolicydTest,
         logging.info('Authentication for {} on keystone IP {}'
                      .format(openrc['OS_USERNAME'], ip))
         if self.tls_rid:
-            openrc['OS_CACERT'] = \
-                openstack_utils.KEYSTONE_LOCAL_CACERT
+            openrc['OS_CACERT'] = openstack_utils.get_cacert()
             openrc['OS_AUTH_URL'] = (
                 openrc['OS_AUTH_URL'].replace('http', 'https'))
         logging.info('keystone IP {}'.format(ip))

zaza/openstack/charm_tests/vault/setup.py

@@ -222,9 +222,8 @@ def validate_ca(cacertificate, application="keystone", port=5000):
     :returns: None
     :rtype: None
     """
-    zaza.model.block_until_file_has_contents(
+    zaza.openstack.utilities.openstack.block_until_ca_exists(
         application,
-        zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT,
         cacertificate.decode().strip())
     vip = (zaza.model.get_application_config(application)
            .get("vip").get("value"))

zaza/openstack/charm_tests/vault/tests.py

@@ -154,9 +154,8 @@ class VaultTest(BaseVaultTest):
 
         test_config = lifecycle_utils.get_charm_config()
         del test_config['target_deploy_status']['vault']
-        zaza.model.block_until_file_has_contents(
+        zaza.openstack.utilities.openstack.block_until_ca_exists(
             'keystone',
-            zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT,
             cacert.decode().strip())
         zaza.model.wait_for_application_states(
             states=test_config.get('target_deploy_status', {}))

zaza/openstack/utilities/openstack.py

@@ -16,7 +16,10 @@
 
 This module contains a number of functions for interacting with OpenStack.
 """
+import collections
+import copy
 import datetime
+import enum
 import io
 import itertools
 import juju_wait
@@ -24,6 +27,7 @@ import logging
 import os
 import paramiko
 import re
+import shutil
 import six
 import subprocess
 import sys
@@ -59,12 +63,14 @@ from keystoneauth1.identity import (
 import zaza.openstack.utilities.cert as cert
 import zaza.utilities.deployment_env as deployment_env
 import zaza.utilities.juju as juju_utils
+import zaza.utilities.maas
 from novaclient import client as novaclient_client
 from neutronclient.v2_0 import client as neutronclient
 from neutronclient.common import exceptions as neutronexceptions
 from octaviaclient.api.v2 import octavia as octaviaclient
 from swiftclient import client as swiftclient
 
+from juju.errors import JujuError
 
 import zaza
 
@@ -177,18 +183,70 @@ WORKLOAD_STATUS_EXCEPTIONS = {
              'ceilometer and gnocchi')}}
 
 # For vault TLS certificates
+CACERT_FILENAME_FORMAT = "{}_juju_ca_cert.crt"
+CERT_PROVIDERS = ['vault']
+LOCAL_CERT_DIR = "tests"
+REMOTE_CERT_DIR = "/usr/local/share/ca-certificates"
 KEYSTONE_CACERT = "keystone_juju_ca_cert.crt"
 KEYSTONE_REMOTE_CACERT = (
     "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT))
-KEYSTONE_LOCAL_CACERT = ("tests/{}".format(KEYSTONE_CACERT))
+KEYSTONE_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, KEYSTONE_CACERT))
+
+
+async def async_block_until_ca_exists(application_name, ca_cert,
+                                      model_name=None, timeout=2700):
+    """Block until a CA cert is on all units of application_name.
+
+    :param application_name: Name of application to check
+    :type application_name: str
+    :param ca_cert: The certificate content.
+    :type ca_cert: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :param timeout: How long in seconds to wait
+    :type timeout: int
+    """
+    async def _check_ca_present(model, ca_files):
+        units = model.applications[application_name].units
+        for ca_file in ca_files:
+            for unit in units:
+                try:
+                    output = await unit.run('cat {}'.format(ca_file))
+                    contents = output.data.get('results').get('Stdout', '')
+                    if ca_cert not in contents:
+                        break
+                # libjuju throws a generic error for connection failure. So we
+                # cannot differentiate between a connectivity issue and a
+                # target file not existing error. For now just assume the
+                # latter.
+                except JujuError:
+                    break
+            else:
+                # The CA was found in `ca_file` on all units.
+                return True
+        else:
+            return False
+    ca_files = await _async_get_remote_ca_cert_file_candidates(
+        application_name,
+        model_name=model_name)
+    async with zaza.model.run_in_model(model_name) as model:
+        await zaza.model.async_block_until(
+            lambda: _check_ca_present(model, ca_files), timeout=timeout)
+
+block_until_ca_exists = zaza.model.sync_wrapper(async_block_until_ca_exists)
 
 
 def get_cacert():
     """Return path to CA Certificate bundle for verification during test.
 
     :returns: Path to CA Certificate bundle or None.
-    :rtype: Optional[str]
+    :rtype: Union[str, None]
     """
+    for _provider in CERT_PROVIDERS:
+        _cert = LOCAL_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format(
+            _provider)
+        if os.path.exists(_cert):
+            return _cert
     if os.path.exists(KEYSTONE_LOCAL_CACERT):
         return KEYSTONE_LOCAL_CACERT
 
@@ -721,6 +779,211 @@ def add_interface_to_netplan(server_name, mac_address):
             model.run_on_unit(unit_name, "sudo netplan apply")
 
 
+class OpenStackNetworkingTopology(enum.Enum):
+    """OpenStack Charms Network Topologies."""
+
+    ML2_OVS = 'ML2+OVS'
+    ML2_OVS_DVR = 'ML2+OVS+DVR'
+    ML2_OVS_DVR_SNAT = 'ML2+OVS+DVR, no dedicated GWs'
+    ML2_OVN = 'ML2+OVN'
+
+
+CharmedOpenStackNetworkingData = collections.namedtuple(
+    'CharmedOpenStackNetworkingData',
+    [
+        'topology',
+        'application_names',
+        'unit_machine_ids',
+        'port_config_key',
+        'other_config',
+    ])
+
+
+def get_charm_networking_data(limit_gws=None):
+    """Inspect Juju model, determine networking topology and return data.
+
+    :param limit_gws: Limit the number of gateways that get a port attached
+    :type limit_gws: Optional[int]
+    :rtype: CharmedOpenStackNetworkingData[
+                OpenStackNetworkingTopology,
+                List[str],
+                Iterator[str],
+                str,
+                Dict[str,str]]
+    :returns: Named Tuple with networking data, example:
+        CharmedOpenStackNetworkingData(
+            OpenStackNetworkingTopology.ML2_OVN,
+            ['ovn-chassis', 'ovn-dedicated-chassis'],
+            ['machine-id-1', 'machine-id-2'],         # generator object
+            'bridge-interface-mappings',
+            {'ovn-bridge-mappings': 'physnet1:br-ex'})
+    :raises: RuntimeError
+    """
+    # Initialize defaults, these will be amended to fit the reality of the
+    # model in the checks below.
+    topology = OpenStackNetworkingTopology.ML2_OVS
+    other_config = {}
+    port_config_key = (
+        'data-port' if not deprecated_external_networking() else 'ext-port')
+    unit_machine_ids = []
+    application_names = []
+
+    if dvr_enabled():
+        if ngw_present():
+            application_names = ['neutron-gateway', 'neutron-openvswitch']
+            topology = OpenStackNetworkingTopology.ML2_OVS_DVR
+        else:
+            application_names = ['neutron-openvswitch']
+            topology = OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT
+        unit_machine_ids = itertools.islice(
+            itertools.chain(
+                get_ovs_uuids(),
+                get_gateway_uuids()),
+            limit_gws)
+    elif ngw_present():
+        unit_machine_ids = itertools.islice(
+            get_gateway_uuids(), limit_gws)
+        application_names = ['neutron-gateway']
+    elif ovn_present():
+        topology = OpenStackNetworkingTopology.ML2_OVN
+        unit_machine_ids = itertools.islice(get_ovn_uuids(), limit_gws)
+        application_names = ['ovn-chassis']
+        try:
+            ovn_dc_name = 'ovn-dedicated-chassis'
+            model.get_application(ovn_dc_name)
+            application_names.append(ovn_dc_name)
+        except KeyError:
+            # ovn-dedicated-chassis not in deployment
+            pass
+        port_config_key = 'bridge-interface-mappings'
+        other_config.update({'ovn-bridge-mappings': 'physnet1:br-ex'})
+    else:
+        raise RuntimeError('Unable to determine charm network topology.')
+
+    return CharmedOpenStackNetworkingData(
+        topology,
+        application_names,
+        unit_machine_ids,
+        port_config_key,
+        other_config)
+
+
+def create_additional_port_for_machines(novaclient, neutronclient, net_id,
+                                        unit_machine_ids,
+                                        add_dataport_to_netplan=False):
+    """Create additional port for machines for use with external networking.
+
+    :param novaclient: Undercloud Authenticated novaclient.
+    :type novaclient: novaclient.Client object
+    :param neutronclient: Undercloud Authenticated neutronclient.
+    :type neutronclient: neutronclient.Client object
+    :param net_id: Network ID to create ports on.
+    :type net_id: string
+    :param unit_machine_ids: Juju provider specific machine IDs for which we
+                             should add ports on.
+    :type unit_machine_ids: Iterator[str]
+    :param add_dataport_to_netplan: Whether the newly created port should be
+                                    added to instance system configuration so
+                                    that it is brought up on instance reboot.
+    :type add_dataport_to_netplan: Optional[bool]
+    :returns: List of MAC addresses for created ports.
+    :rtype: List[str]
+    :raises: RuntimeError
+    """
+    eligible_machines = 0
+    for uuid in unit_machine_ids:
+        eligible_machines += 1
+        server = novaclient.servers.get(uuid)
+        ext_port_name = "{}_ext-port".format(server.name)
+        for port in neutronclient.list_ports(device_id=server.id)['ports']:
+            if port['name'] == ext_port_name:
+                logging.warning(
+                    'Instance {} already has additional port, skipping.'
+                    .format(server.id))
+                break
+        else:
+            logging.info('Attaching additional port to instance ("{}"), '
+                         'connected to net id: {}'
+                         .format(uuid, net_id))
+            body_value = {
+                "port": {
+                    "admin_state_up": True,
+                    "name": ext_port_name,
+                    "network_id": net_id,
+                    "port_security_enabled": False,
+                }
+            }
+            port = neutronclient.create_port(body=body_value)
+            server.interface_attach(port_id=port['port']['id'],
+                                    net_id=None, fixed_ip=None)
+            if add_dataport_to_netplan:
+                mac_address = get_mac_from_port(port, neutronclient)
+                add_interface_to_netplan(server.name,
+                                         mac_address=mac_address)
+    if not eligible_machines:
+        # NOTE: unit_machine_ids may be an iterator so testing it for contents
+        # or length prior to iterating over it is futile.
+        raise RuntimeError('Unable to determine UUIDs for machines to attach '
+                           'external networking to.')
+
+    # Retrieve the just created ports from Neutron so that we can provide our
+    # caller with their MAC addresses.
+    return [
+        port['mac_address']
+        for port in neutronclient.list_ports(network_id=net_id)['ports']
+        if 'ext-port' in port['name']
+    ]
+
+
+def configure_networking_charms(networking_data, macs, use_juju_wait=True):
+    """Configure external networking for networking charms.
+
+    :param networking_data: Data on networking charm topology.
+    :type networking_data: CharmedOpenStackNetworkingData
+    :param macs: MAC addresses of ports for use with external networking.
+    :type macs: Iterator[str]
+    :param use_juju_wait: Whether to use juju wait to wait for the model to
+        settle once the gateway has been configured. Default is True
+    :type use_juju_wait: Optional[bool]
+    """
+    br_mac_fmt = 'br-ex:{}' if not deprecated_external_networking() else '{}'
+    br_mac = [
+        br_mac_fmt.format(mac)
+        for mac in macs
+    ]
+
+    config = copy.deepcopy(networking_data.other_config)
+    config.update({networking_data.port_config_key: ' '.join(sorted(br_mac))})
+
+    for application_name in networking_data.application_names:
+        logging.info('Setting {} on {}'.format(
+            config, application_name))
+        current_data_port = get_application_config_option(
+            application_name,
+            networking_data.port_config_key)
+        if current_data_port == config[networking_data.port_config_key]:
+            logging.info('Config already set to value')
+            return
+
+        model.set_application_config(
+            application_name,
+            configuration=config)
+    # NOTE(fnordahl): We are stuck with juju_wait until we figure out how
+    # to deal with all the non ['active', 'idle', 'Unit is ready.']
+    # workload/agent states and msgs that our mojo specs are exposed to.
+    if use_juju_wait:
+        juju_wait.wait(wait_for_workload=True)
+    else:
+        zaza.model.wait_for_agent_status()
+        # TODO: shouldn't access get_charm_config() here as it relies on
+        # ./tests/tests.yaml existing by default (regardless of the
+        # fatal=False) ... it's not great design.
+        test_config = zaza.charm_lifecycle.utils.get_charm_config(
+            fatal=False)
+        zaza.model.wait_for_application_states(
+            states=test_config.get('target_deploy_status', {}))
+
+
 def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
                                add_dataport_to_netplan=False,
                                limit_gws=None,
@@ -739,123 +1002,46 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
         settle once the gateway has been configured. Default is True
     :type use_juju_wait: boolean
     """
-    deprecated_extnet_mode = deprecated_external_networking()
-
-    port_config_key = 'data-port'
-    if deprecated_extnet_mode:
-        port_config_key = 'ext-port'
-
-    config = {}
-    if dvr_enabled():
-        uuids = itertools.islice(itertools.chain(get_ovs_uuids(),
-                                                 get_gateway_uuids()),
-                                 limit_gws)
+    networking_data = get_charm_networking_data(limit_gws=limit_gws)
+    if networking_data.topology in (
+            OpenStackNetworkingTopology.ML2_OVS_DVR,
+            OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT):
         # If dvr, do not attempt to persist nic in netplan
         # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78
         add_dataport_to_netplan = False
-        application_names = ['neutron-openvswitch']
-        try:
-            ngw = 'neutron-gateway'
-            model.get_application(ngw)
-            application_names.append(ngw)
-        except KeyError:
-            # neutron-gateway not in deployment
-            pass
-    elif ngw_present():
-        uuids = itertools.islice(get_gateway_uuids(), limit_gws)
-        application_names = ['neutron-gateway']
-    elif ovn_present():
-        uuids = itertools.islice(get_ovn_uuids(), limit_gws)
-        application_names = ['ovn-chassis']
-        try:
-            ovn_dc_name = 'ovn-dedicated-chassis'
-            model.get_application(ovn_dc_name)
-            application_names.append(ovn_dc_name)
-        except KeyError:
-            # ovn-dedicated-chassis not in deployment
-            pass
-        port_config_key = 'bridge-interface-mappings'
-        config.update({'ovn-bridge-mappings': 'physnet1:br-ex'})
-        add_dataport_to_netplan = True
-    else:
-        raise RuntimeError('Unable to determine charm network topology.')
 
     if not net_id:
         net_id = get_admin_net(neutronclient)['id']
 
-    ports_created = 0
-    for uuid in uuids:
-        server = novaclient.servers.get(uuid)
-        ext_port_name = "{}_ext-port".format(server.name)
-        for port in neutronclient.list_ports(device_id=server.id)['ports']:
-            if port['name'] == ext_port_name:
-                logging.warning(
-                    'Neutron Gateway already has additional port')
-                break
-        else:
-            logging.info('Attaching additional port to instance ("{}"), '
-                         'connected to net id: {}'
-                         .format(uuid, net_id))
-            body_value = {
-                "port": {
-                    "admin_state_up": True,
-                    "name": ext_port_name,
-                    "network_id": net_id,
-                    "port_security_enabled": False,
-                }
-            }
-            port = neutronclient.create_port(body=body_value)
-            ports_created += 1
-            server.interface_attach(port_id=port['port']['id'],
-                                    net_id=None, fixed_ip=None)
-            if add_dataport_to_netplan:
-                mac_address = get_mac_from_port(port, neutronclient)
-                add_interface_to_netplan(server.name,
-                                         mac_address=mac_address)
-    if not ports_created:
-        # NOTE: uuids is an iterator so testing it for contents or length prior
-        # to iterating over it is futile.
-        raise RuntimeError('Unable to determine UUIDs for machines to attach '
-                           'external networking to.')
+    macs = create_additional_port_for_machines(
+        novaclient, neutronclient, net_id, networking_data.unit_machine_ids,
+        add_dataport_to_netplan)
 
-    ext_br_macs = []
-    for port in neutronclient.list_ports(network_id=net_id)['ports']:
-        if 'ext-port' in port['name']:
-            if deprecated_extnet_mode:
-                ext_br_macs.append(port['mac_address'])
-            else:
-                ext_br_macs.append('br-ex:{}'.format(port['mac_address']))
-    ext_br_macs.sort()
-    ext_br_macs_str = ' '.join(ext_br_macs)
+    if macs:
+        configure_networking_charms(
+            networking_data, macs, use_juju_wait=use_juju_wait)
 
-    if ext_br_macs:
-        config.update({port_config_key: ext_br_macs_str})
-        for application_name in application_names:
-            logging.info('Setting {} on {}'.format(
-                config, application_name))
-            current_data_port = get_application_config_option(application_name,
-                                                              port_config_key)
-            if current_data_port == ext_br_macs_str:
-                logging.info('Config already set to value')
-                return
 
-            model.set_application_config(
-                application_name,
-                configuration=config)
-        # NOTE(fnordahl): We are stuck with juju_wait until we figure out how
-        # to deal with all the non ['active', 'idle', 'Unit is ready.']
-        # workload/agent states and msgs that our mojo specs are exposed to.
-        if use_juju_wait:
-            juju_wait.wait(wait_for_workload=True)
-        else:
-            zaza.model.wait_for_agent_status()
-            # TODO: shouldn't access get_charm_config() here as it relies on
-            # ./tests/tests.yaml existing by default (regardless of the
-            # fatal=False) ... it's not great design.
-            test_config = zaza.charm_lifecycle.utils.get_charm_config(
-                fatal=False)
-            zaza.model.wait_for_application_states(
-                states=test_config.get('target_deploy_status', {}))
+def configure_charmed_openstack_on_maas(network_config, limit_gws=None):
+    """Configure networking charms for charm-based OVS config on MAAS provider.
+
+    :param network_config: Network configuration as provided in environment.
+    :type network_config: Dict[str]
+    :param limit_gws: Limit the number of gateways that get a port attached
+    :type limit_gws: Optional[int]
+    """
+    networking_data = get_charm_networking_data(limit_gws=limit_gws)
+    macs = [
+        mim.mac
+        for mim in zaza.utilities.maas.get_macs_from_cidr(
+            zaza.utilities.maas.get_maas_client_from_juju_cloud_data(
+                zaza.model.get_cloud_data()),
+            network_config['external_net_cidr'],
+            link_mode=zaza.utilities.maas.LinkMode.LINK_UP)
+    ]
+    if macs:
+        configure_networking_charms(
+            networking_data, macs, use_juju_wait=False)
 
 
 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
@@ -1818,29 +2004,83 @@ def get_overcloud_auth(address=None, model_name=None):
             'OS_PROJECT_DOMAIN_NAME': 'admin_domain',
             'API_VERSION': 3,
         }
-    if tls_rid:
-        unit = model.get_first_unit_name('keystone', model_name=model_name)
-
-        # ensure that the path to put the local cacert in actually exists.  The
-        # assumption that 'tests/' exists for, say, mojo is false.
-        # Needed due to:
-        # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2
-        _dir = os.path.dirname(KEYSTONE_LOCAL_CACERT)
-        if not os.path.exists(_dir):
-            os.makedirs(_dir)
-
-        model.scp_from_unit(
-            unit,
-            KEYSTONE_REMOTE_CACERT,
-            KEYSTONE_LOCAL_CACERT)
-
-        if os.path.exists(KEYSTONE_LOCAL_CACERT):
-            os.chmod(KEYSTONE_LOCAL_CACERT, 0o644)
-            auth_settings['OS_CACERT'] = KEYSTONE_LOCAL_CACERT
+    local_ca_cert = get_remote_ca_cert_file('keystone', model_name=model_name)
+    if local_ca_cert:
+        auth_settings['OS_CACERT'] = local_ca_cert
 
     return auth_settings
 
 
+async def _async_get_remote_ca_cert_file_candidates(application,
+                                                    model_name=None):
+    """Return a list of possible remote CA file names.
+
+    :param application: Name of application to examine.
+    :type application: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :returns: List of paths to possible ca files.
+    :rtype: List[str]
+    """
+    cert_files = []
+    for _provider in CERT_PROVIDERS:
+        tls_rid = await model.async_get_relation_id(
+            application,
+            _provider,
+            model_name=model_name,
+            remote_interface_name='certificates')
+        if tls_rid:
+            cert_files.append(
+                REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format(
+                    _provider))
+    cert_files.append(KEYSTONE_REMOTE_CACERT)
+    return cert_files
+
+_get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper(
+    _async_get_remote_ca_cert_file_candidates)
+
+
+def get_remote_ca_cert_file(application, model_name=None):
+    """Collect CA certificate from application.
+
+    :param application: Name of application to collect file from.
+    :type application: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :returns: Path to cafile
+    :rtype: str
+    """
+    unit = model.get_first_unit_name(application, model_name=model_name)
+    local_cert_file = None
+    cert_files = _get_remote_ca_cert_file_candidates(
+        application,
+        model_name=model_name)
+    for cert_file in cert_files:
+        _local_cert_file = "{}/{}".format(
+            LOCAL_CERT_DIR,
+            os.path.basename(cert_file))
+        with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca:
+            try:
+                model.scp_from_unit(
+                    unit,
+                    cert_file,
+                    _tmp_ca.name)
+            except JujuError:
+                continue
+            # ensure that the path to put the local cacert in actually exists.
+            # The assumption that 'tests/' exists for, say, mojo is false.
+            # Needed due to:
+            # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2
+            _dir = os.path.dirname(_local_cert_file)
+            if not os.path.exists(_dir):
+                os.makedirs(_dir)
+            shutil.move(_tmp_ca.name, _local_cert_file)
+            os.chmod(_local_cert_file, 0o644)
+            local_cert_file = _local_cert_file
+            break
+    return local_cert_file
+
+
 def get_urllib_opener():
     """Create a urllib opener taking into account proxy settings.