Merge branch 'master' into bug/1897809

Change-Id: Ib36cf72fe20aa2b46a80f2d71bcd1400fd70fb8e

.github/workflows/tox.yaml

@@ -0,0 +1,27 @@
+name: Python package
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox tox-gh-actions
+    - name: Lint with tox
+      run: tox -e pep8
+    - name: Test with tox
+      run: tox -e py${{ matrix.python-version }}

requirements.txt

@@ -1,3 +1,7 @@
+# pin lxml < 4.6.3 for py35 as no wheels exist for 4.6.3 (deprecated platform)
+# This is necessary for Xenial builders
+# BUG: https://github.com/openstack-charmers/zaza-openstack-tests/issues/530
+lxml<4.6.3
 aiounittest
 async_generator
 boto3
@@ -29,6 +33,7 @@ python-ceilometerclient
 python-cinderclient
 python-glanceclient
 python-heatclient
+python-ironicclient
 python-keystoneclient
 python-manilaclient
 python-neutronclient
@@ -43,3 +48,8 @@ paramiko
 sphinx
 sphinxcontrib-asyncio
 git+https://github.com/openstack-charmers/zaza#egg=zaza
+
+# Newer versions require a Rust compiler to build, see
+# * https://github.com/openstack-charmers/zaza/issues/421
+# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html
+cryptography<3.4

setup.py

@@ -28,7 +28,12 @@ install_require = [
     'futurist<2.0.0',
     'async_generator',
     'boto3',
-    'cryptography',
+
+    # Newer versions require a Rust compiler to build, see
+    # * https://github.com/openstack-charmers/zaza/issues/421
+    # * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html
+    'cryptography<3.4',
+
     'dnspython',
     'hvac<0.7.0',
     'jinja2',
@@ -44,6 +49,7 @@ install_require = [
     'python-barbicanclient>=4.0.1,<5.0.0',
     'python-designateclient>=1.5,<3.0.0',
     'python-heatclient<2.0.0',
+    'python-ironicclient',
     'python-glanceclient<3.0.0',
     'python-keystoneclient<3.22.0',
     'python-manilaclient<2.0.0',

tox.ini

@@ -1,6 +1,22 @@
 [tox]
 envlist = pep8, py3
 skipsdist = True
+# NOTE: Avoid build/test env pollution by not enabling sitepackages.
+sitepackages = False
+# NOTE: Avoid false positives by not skipping missing interpreters.
+skip_missing_interpreters = False
+# NOTES:
+# * We avoid the new dependency resolver by pinning pip < 20.3, see
+#   https://github.com/pypa/pip/issues/9187
+# * Pinning dependencies requires tox >= 3.2.0, see
+#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
+# * It is also necessary to pin virtualenv as a newer virtualenv would still
+#   lead to fetching the latest pip in the func* tox targets, see
+#   https://stackoverflow.com/a/38133283
+requires = pip < 20.3
+           virtualenv < 20.0
+# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
+minversion = 3.2.0
 
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
@@ -14,6 +30,26 @@ commands = nosetests --with-coverage --cover-package=zaza.openstack {posargs} {t
 basepython = python3
 deps = -r{toxinidir}/requirements.txt
 
+[testenv:py3.5]
+basepython = python3.5
+deps = -r{toxinidir}/requirements.txt
+
+[testenv:py3.6]
+basepython = python3.6
+deps = -r{toxinidir}/requirements.txt
+
+[testenv:py3.7]
+basepython = python3.7
+deps = -r{toxinidir}/requirements.txt
+
+[testenv:py3.8]
+basepython = python3.8
+deps = -r{toxinidir}/requirements.txt
+
+[testenv:py3.9]
+basepython = python3.9
+deps = -r{toxinidir}/requirements.txt
+
 [testenv:pep8]
 basepython = python3
 deps = -r{toxinidir}/requirements.txt

unit_tests/__init__.py

@@ -11,3 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import sys
+import unittest.mock as mock
+
+sys.modules['zaza.utilities.maas'] = mock.MagicMock()

unit_tests/charm_tests/test_utils.py

@@ -97,7 +97,7 @@ class TestBaseCharmTest(ut_utils.BaseTestCase):
             self.set_application_config.reset_mock()
         self.assertFalse(self.set_application_config.called)
         self.reset_application_config.assert_called_once_with(
-            'anApp', alterna_config.keys(), model_name='aModel')
+            'anApp', list(alterna_config.keys()), model_name='aModel')
         self.wait_for_application_states.assert_has_calls([
             mock.call(model_name='aModel', states={}),
             mock.call(model_name='aModel', states={}),
@@ -125,6 +125,66 @@ class TestBaseCharmTest(ut_utils.BaseTestCase):
         self.assertFalse(self.wait_for_agent_status.called)
         self.assertFalse(self.wait_for_application_states.called)
 
+    def test_separate_non_string_config(self):
+        intended_cfg_keys = ['foo2', 'foo3', 'foo4', 'foo5']
+        current_config_mock = {
+            'foo2': None,
+            'foo3': 'old_bar3',
+            'foo4': None,
+            'foo5': 'old_bar5',
+        }
+        self.patch_target('config_current')
+        self.config_current.return_value = current_config_mock
+        non_string_type_keys = ['foo2', 'foo3', 'foo4']
+        expected_result_filtered = {
+            'foo3': 'old_bar3',
+            'foo5': 'old_bar5',
+        }
+        expected_result_special = {
+            'foo2': None,
+            'foo4': None,
+        }
+        current, non_string = (
+            self.target.config_current_separate_non_string_type_keys(
+                non_string_type_keys, intended_cfg_keys, 'application_name')
+        )
+
+        self.assertEqual(expected_result_filtered, current)
+        self.assertEqual(expected_result_special, non_string)
+
+        self.config_current.assert_called_once_with(
+            'application_name', intended_cfg_keys)
+
+    def test_separate_special_config_None_params(self):
+        current_config_mock = {
+            'foo1': 'old_bar1',
+            'foo2': None,
+            'foo3': 'old_bar3',
+            'foo4': None,
+            'foo5': 'old_bar5',
+        }
+        self.patch_target('config_current')
+        self.config_current.return_value = current_config_mock
+        non_string_type_keys = ['foo2', 'foo3', 'foo4']
+        expected_result_filtered = {
+            'foo1': 'old_bar1',
+            'foo3': 'old_bar3',
+            'foo5': 'old_bar5',
+        }
+        expected_result_special = {
+            'foo2': None,
+            'foo4': None,
+        }
+        current, non_string = (
+            self.target.config_current_separate_non_string_type_keys(
+                non_string_type_keys)
+        )
+
+        self.assertEqual(expected_result_filtered, current)
+        self.assertEqual(expected_result_special, non_string)
+
+        self.config_current.assert_called_once_with(None, None)
+
 
 class TestOpenStackBaseTest(ut_utils.BaseTestCase):
 

unit_tests/utilities/test_utilities.py

@@ -0,0 +1,188 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+import unit_tests.utils as ut_utils
+
+import zaza.openstack.utilities as utilities
+
+
+class SomeException(Exception):
+    pass
+
+
+class SomeException2(Exception):
+    pass
+
+
+class SomeException3(Exception):
+    pass
+
+
+class TestObjectRetrierWraps(ut_utils.BaseTestCase):
+
+    def test_object_wrap(self):
+
+        class A:
+
+            def func(self, a, b=1):
+                return a + b
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(a)
+        self.assertEqual(wrapped_a.func(3), 4)
+
+    def test_object_multilevel_wrap(self):
+
+        class A:
+
+            def f1(self, a, b):
+                return a * b
+
+        class B:
+
+            @property
+            def f2(self):
+
+                return A()
+
+        b = B()
+        wrapped_b = utilities.ObjectRetrierWraps(b)
+        self.assertEqual(wrapped_b.f2.f1(5, 6), 30)
+
+    def test_object_wrap_number(self):
+
+        class A:
+
+            class_a = 5
+
+            def __init__(self):
+                self.instance_a = 10
+
+            def f1(self, a, b):
+                return a * b
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(a)
+        self.assertEqual(wrapped_a.class_a, 5)
+        self.assertEqual(wrapped_a.instance_a, 10)
+
+    @mock.patch("time.sleep")
+    def test_object_wrap_exception(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        # retry on a specific exception
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=1, retry_exceptions=[SomeException])
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        mock_sleep.assert_called_once_with(5)
+
+        # also retry on any exception if none specified
+        wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=1)
+        mock_sleep.reset_mock()
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        mock_sleep.assert_called_once_with(5)
+
+        # no retry if exception isn't listed.
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=1, retry_exceptions=[SomeException2])
+        mock_sleep.reset_mock()
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        mock_sleep.assert_not_called()
+
+    @mock.patch("time.sleep")
+    def test_log_called(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        mock_log = mock.Mock()
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=1, log=mock_log)
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        # there should be two calls; one for the single retry and one for the
+        # failure.
+        self.assertEqual(mock_log.call_count, 2)
+
+    @mock.patch("time.sleep")
+    def test_back_off_maximum(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=3, backoff=2)
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+        # Note third call hits maximum wait time of 15.
+        mock_sleep.assert_has_calls([mock.call(5),
+                                     mock.call(10),
+                                     mock.call(15)])
+
+    @mock.patch("time.sleep")
+    def test_total_wait(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=3, total_wait=9)
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+        # Note only two calls, as total wait is 9, so a 3rd retry would exceed
+        # that.
+        mock_sleep.assert_has_calls([mock.call(5),
+                                     mock.call(5)])
+
+    @mock.patch("time.sleep")
+    def test_retry_on_connect_failure(self, mock_sleep):
+
+        class A:
+
+            def func1(self):
+                raise SomeException()
+
+            def func2(self):
+                raise utilities.ConnectFailure()
+
+        a = A()
+        wrapped_a = utilities.retry_on_connect_failure(a, num_retries=2)
+        with self.assertRaises(SomeException):
+            wrapped_a.func1()
+        mock_sleep.assert_not_called()
+        with self.assertRaises(utilities.ConnectFailure):
+            wrapped_a.func2()
+        mock_sleep.assert_has_calls([mock.call(5)])

unit_tests/utilities/test_zaza_utilities_ceph.py

@@ -118,7 +118,7 @@ class TestCephUtils(ut_utils.BaseTestCase):
                                     model_name='amodel')
 
     def test_pools_from_broker_req(self):
-        self.patch_object(ceph_utils.zaza_juju, 'get_relation_from_unit')
+        self.patch_object(ceph_utils.juju_utils, 'get_relation_from_unit')
         self.get_relation_from_unit.return_value = {
             'broker_req': (
                 '{"api-version": 1, "ops": ['

unit_tests/utilities/test_zaza_utilities_openstack.py

@@ -17,6 +17,8 @@ import datetime
 import io
 import mock
 import subprocess
+import sys
+import unittest
 import tenacity
 
 import unit_tests.utils as ut_utils
@@ -191,6 +193,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.patch_object(openstack_utils, 'get_application_config_option')
         self.patch_object(openstack_utils, 'get_keystone_ip')
         self.patch_object(openstack_utils, "get_current_os_versions")
+        self.patch_object(openstack_utils, "get_remote_ca_cert_file")
         self.patch_object(openstack_utils.juju_utils, 'leader_get')
         if tls_relation:
             self.patch_object(openstack_utils.model, "scp_from_unit")
@@ -204,6 +207,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.get_relation_id.return_value = None
         self.get_application_config_option.return_value = None
         self.leader_get.return_value = 'openstack'
+        self.get_remote_ca_cert_file.return_value = None
         if tls_relation or ssl_cert:
             port = 35357
             transport = 'https'
@@ -245,7 +249,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
                 'API_VERSION': 3,
             }
         if tls_relation:
-            expect['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT
+            self.get_remote_ca_cert_file.return_value = '/tmp/a.cert'
+            expect['OS_CACERT'] = '/tmp/a.cert'
         self.assertEqual(openstack_utils.get_overcloud_auth(),
                          expect)
 
@@ -289,6 +294,15 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         openstack_utils.get_undercloud_keystone_session()
         self.get_keystone_session.assert_called_once_with(_auth, verify=None)
 
+    def test_get_nova_session_client(self):
+        session_mock = mock.MagicMock()
+        self.patch_object(openstack_utils.novaclient_client, "Client")
+        openstack_utils.get_nova_session_client(session_mock)
+        self.Client.assert_called_once_with(2, session=session_mock)
+        self.Client.reset_mock()
+        openstack_utils.get_nova_session_client(session_mock, version=2.56)
+        self.Client.assert_called_once_with(2.56, session=session_mock)
+
     def test_get_urllib_opener(self):
         self.patch_object(openstack_utils.urllib.request, "ProxyHandler")
         self.patch_object(openstack_utils.urllib.request, "HTTPHandler")
@@ -369,12 +383,15 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
                 'e01df65a')
 
     def test__resource_reaches_status_bespoke(self):
+        client_mock = mock.MagicMock()
         resource_mock = mock.MagicMock()
-        resource_mock.get.return_value = mock.MagicMock(status='readyish')
+        resource_mock.special_status = 'readyish'
+        client_mock.get.return_value = resource_mock
         openstack_utils._resource_reaches_status(
-            resource_mock,
+            client_mock,
             'e01df65a',
-            'readyish')
+            'readyish',
+            resource_attribute='special_status')
 
     def test__resource_reaches_status_bespoke_fail(self):
         resource_mock = mock.MagicMock()
@@ -504,7 +521,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
             glance_mock.images.upload.assert_called_once_with(
                 '9d1125af',
                 f(),
-            )
+                backend=None)
             self.resource_reaches_status.assert_called_once_with(
                 glance_mock.images,
                 '9d1125af',
@@ -529,7 +546,12 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.upload_image_to_glance.assert_called_once_with(
             glance_mock,
             'wibbly/c.img',
-            'bob')
+            'bob',
+            backend=None,
+            disk_format='qcow2',
+            visibility='public',
+            container_format='bare',
+            force_import=False)
 
     def test_create_image_pass_directory(self):
         glance_mock = mock.MagicMock()
@@ -549,7 +571,12 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.upload_image_to_glance.assert_called_once_with(
             glance_mock,
             'tests/c.img',
-            'bob')
+            'bob',
+            backend=None,
+            disk_format='qcow2',
+            visibility='public',
+            container_format='bare',
+            force_import=False)
         self.gettempdir.assert_not_called()
 
     def test_create_ssh_key(self):
@@ -859,7 +886,24 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         result = openstack_utils.get_current_os_release_pair()
         self.assertEqual(expected, result)
 
-    def test_get_openstack_release(self):
+    def test_get_current_os_versions(self):
+        self.patch_object(openstack_utils, "get_openstack_release")
+        self.patch_object(openstack_utils.generic_utils, "get_pkg_version")
+
+        # Pre-Wallaby scenario where openstack-release package isn't installed
+        self.get_openstack_release.return_value = None
+        self.get_pkg_version.return_value = '18.0.0'
+        expected = {'keystone': 'victoria'}
+        result = openstack_utils.get_current_os_versions('keystone')
+        self.assertEqual(expected, result)
+
+        # Wallaby+ scenario where openstack-release package is installed
+        self.get_openstack_release.return_value = 'wallaby'
+        expected = {'keystone': 'wallaby'}
+        result = openstack_utils.get_current_os_versions('keystone')
+        self.assertEqual(expected, result)
+
+    def test_get_os_release(self):
         self.patch(
             'zaza.openstack.utilities.openstack.get_current_os_release_pair',
             new_callable=mock.MagicMock(),
@@ -888,6 +932,14 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         release_comp = xenial_queens > xenial_mitaka
         self.assertTrue(release_comp)
 
+        # Check specifying an application
+        self._get_os_rel_pair.reset_mock()
+        self._get_os_rel_pair.return_value = 'xenial_mitaka'
+        expected = 4
+        result = openstack_utils.get_os_release(application='myapp')
+        self.assertEqual(expected, result)
+        self._get_os_rel_pair.assert_called_once_with(application='myapp')
+
     def test_get_keystone_api_version(self):
         self.patch_object(openstack_utils, "get_current_os_versions")
         self.patch_object(openstack_utils, "get_application_config_option")
@@ -903,6 +955,23 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.get_application_config_option.return_value = None
         self.assertEqual(openstack_utils.get_keystone_api_version(), 3)
 
+    def test_get_openstack_release(self):
+        self.patch_object(openstack_utils.model, "get_units")
+        self.patch_object(openstack_utils.juju_utils, "remote_run")
+
+        # Test pre-Wallaby behavior where openstack-release pkg isn't installed
+        self.get_units.return_value = []
+        self.remote_run.return_value = "OPENSTACK_CODENAME=wallaby "
+
+        # Test Wallaby+ behavior where openstack-release package is installed
+        unit1 = mock.MagicMock()
+        unit1.entity_id = 1
+        self.get_units.return_value = [unit1]
+        self.remote_run.return_value = "OPENSTACK_CODENAME=wallaby "
+
+        result = openstack_utils.get_openstack_release("application", "model")
+        self.assertEqual(result, "wallaby")
+
     def test_get_project_id(self):
         # No domain
         self.patch_object(openstack_utils, "get_keystone_api_version")
@@ -1234,34 +1303,230 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
         self.get_application.side_effect = KeyError
         self.assertFalse(openstack_utils.ngw_present())
 
-    def test_configure_gateway_ext_port(self):
-        # FIXME: this is not a complete unit test for the function as one did
-        # not exist at all I'm adding this to test one bit and we'll add more
-        # as we go.
+    def test_get_charm_networking_data(self):
         self.patch_object(openstack_utils, 'deprecated_external_networking')
         self.patch_object(openstack_utils, 'dvr_enabled')
         self.patch_object(openstack_utils, 'ovn_present')
         self.patch_object(openstack_utils, 'ngw_present')
+        self.patch_object(openstack_utils, 'get_ovs_uuids')
         self.patch_object(openstack_utils, 'get_gateway_uuids')
-        self.patch_object(openstack_utils, 'get_admin_net')
+        self.patch_object(openstack_utils, 'get_ovn_uuids')
+        self.patch_object(openstack_utils.model, 'get_application')
         self.dvr_enabled.return_value = False
         self.ovn_present.return_value = False
-        self.ngw_present.return_value = True
-        self.get_admin_net.return_value = {'id': 'fakeid'}
+        self.ngw_present.return_value = False
+        self.get_ovs_uuids.return_value = []
+        self.get_gateway_uuids.return_value = []
+        self.get_ovn_uuids.return_value = []
+        self.get_application.side_effect = KeyError
 
-        novaclient = mock.MagicMock()
-        neutronclient = mock.MagicMock()
-
-        def _fake_empty_generator(empty=True):
-            if empty:
-                return
-            yield
-
-        self.get_gateway_uuids.side_effect = _fake_empty_generator
         with self.assertRaises(RuntimeError):
-            openstack_utils.configure_gateway_ext_port(
-                novaclient, neutronclient)
-        # provide a uuid and check that we don't raise RuntimeError
-        self.get_gateway_uuids.side_effect = ['fake-uuid']
-        openstack_utils.configure_gateway_ext_port(
-            novaclient, neutronclient)
+            openstack_utils.get_charm_networking_data()
+        self.ngw_present.return_value = True
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVS,
+                ['neutron-gateway'],
+                mock.ANY,
+                'data-port',
+                {}))
+        self.dvr_enabled.return_value = True
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR,
+                ['neutron-gateway', 'neutron-openvswitch'],
+                mock.ANY,
+                'data-port',
+                {}))
+        self.ngw_present.return_value = False
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT,
+                ['neutron-openvswitch'],
+                mock.ANY,
+                'data-port',
+                {}))
+        self.dvr_enabled.return_value = False
+        self.ovn_present.return_value = True
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVN,
+                ['ovn-chassis'],
+                mock.ANY,
+                'bridge-interface-mappings',
+                {'ovn-bridge-mappings': 'physnet1:br-ex'}))
+        self.get_application.side_effect = None
+        self.assertEquals(
+            openstack_utils.get_charm_networking_data(),
+            openstack_utils.CharmedOpenStackNetworkingData(
+                openstack_utils.OpenStackNetworkingTopology.ML2_OVN,
+                ['ovn-chassis', 'ovn-dedicated-chassis'],
+                mock.ANY,
+                'bridge-interface-mappings',
+                {'ovn-bridge-mappings': 'physnet1:br-ex'}))
+
+    def test_get_cacert_absolute_path(self):
+        self.patch_object(openstack_utils.deployment_env, 'get_tmpdir')
+        self.get_tmpdir.return_value = '/tmp/default'
+        self.assertEqual(
+            openstack_utils.get_cacert_absolute_path('filename'),
+            '/tmp/default/filename')
+
+    def test_get_cacert(self):
+        self.patch_object(openstack_utils.deployment_env, 'get_tmpdir')
+        self.get_tmpdir.return_value = '/tmp/default'
+        self.patch_object(openstack_utils.os.path, 'exists')
+        results = {
+            '/tmp/default/vault_juju_ca_cert.crt': True}
+        self.exists.side_effect = lambda x: results[x]
+        self.assertEqual(
+            openstack_utils.get_cacert(),
+            '/tmp/default/vault_juju_ca_cert.crt')
+
+        results = {
+            '/tmp/default/vault_juju_ca_cert.crt': False,
+            '/tmp/default/keystone_juju_ca_cert.crt': True}
+        self.assertEqual(
+            openstack_utils.get_cacert(),
+            '/tmp/default/keystone_juju_ca_cert.crt')
+
+        results = {
+            '/tmp/default/vault_juju_ca_cert.crt': False,
+            '/tmp/default/keystone_juju_ca_cert.crt': False}
+        self.assertIsNone(openstack_utils.get_cacert())
+
+    def test_get_remote_ca_cert_file(self):
+        self.patch_object(openstack_utils.model, 'get_first_unit_name')
+        self.patch_object(
+            openstack_utils,
+            '_get_remote_ca_cert_file_candidates')
+        self.patch_object(openstack_utils.model, 'scp_from_unit')
+        self.patch_object(openstack_utils.os.path, 'exists')
+        self.patch_object(openstack_utils.shutil, 'move')
+        self.patch_object(openstack_utils.os, 'chmod')
+        self.patch_object(openstack_utils.tempfile, 'NamedTemporaryFile')
+        self.patch_object(openstack_utils.deployment_env, 'get_tmpdir')
+        self.get_tmpdir.return_value = '/tmp/default'
+        enter_mock = mock.MagicMock()
+        enter_mock.__enter__.return_value.name = 'tempfilename'
+        self.NamedTemporaryFile.return_value = enter_mock
+        self.get_first_unit_name.return_value = 'neutron-api/0'
+        self._get_remote_ca_cert_file_candidates.return_value = [
+            '/tmp/ca1.cert']
+        self.exists.return_value = True
+
+        openstack_utils.get_remote_ca_cert_file('neutron-api')
+        self.scp_from_unit.assert_called_once_with(
+            'neutron-api/0',
+            '/tmp/ca1.cert',
+            'tempfilename')
+        self.chmod.assert_called_once_with('/tmp/default/ca1.cert', 0o644)
+        self.move.assert_called_once_with(
+            'tempfilename', '/tmp/default/ca1.cert')
+
+
+class TestAsyncOpenstackUtils(ut_utils.AioTestCase):
+
+    def setUp(self):
+        super(TestAsyncOpenstackUtils, self).setUp()
+        if sys.version_info < (3, 6, 0):
+            raise unittest.SkipTest("Can't AsyncMock in py35")
+        model_mock = mock.MagicMock()
+        test_mock = mock.MagicMock()
+
+        class AsyncContextManagerMock(test_mock):
+            async def __aenter__(self):
+                return self
+
+            async def __aexit__(self, *args):
+                pass
+
+        self.model_mock = model_mock
+        self.patch_object(openstack_utils.zaza.model, "async_block_until")
+
+        async def _block_until(f, timeout):
+            # Store the result of the call to _check_ca_present to validate
+            # tests
+            self.result = await f()
+        self.async_block_until.side_effect = _block_until
+        self.patch('zaza.model.run_in_model', name='_run_in_model')
+        self._run_in_model.return_value = AsyncContextManagerMock
+        self._run_in_model().__aenter__.return_value = self.model_mock
+
+    async def test_async_block_until_ca_exists(self):
+        def _get_action_output(stdout, code, stderr=None):
+            stderr = stderr or ''
+            action = mock.MagicMock()
+            action.data = {
+                'results': {
+                    'Code': code,
+                    'Stderr': stderr,
+                    'Stdout': stdout}}
+            return action
+        results = {
+            '/tmp/missing.cert': _get_action_output(
+                '',
+                '1',
+                'cat: /tmp/missing.cert: No such file or directory'),
+            '/tmp/good.cert': _get_action_output('CERTIFICATE', '0')}
+
+        async def _run(command, timeout=None):
+            return results[command.split()[-1]]
+        self.unit1 = mock.MagicMock()
+        self.unit2 = mock.MagicMock()
+        self.unit2.run.side_effect = _run
+        self.unit1.run.side_effect = _run
+        self.units = [self.unit1, self.unit2]
+        _units = mock.MagicMock()
+        _units.units = self.units
+        self.model_mock.applications = {
+            'keystone': _units
+        }
+        self.patch_object(
+            openstack_utils,
+            "_async_get_remote_ca_cert_file_candidates")
+
+        # Test a missing cert then a good cert.
+        self._async_get_remote_ca_cert_file_candidates.return_value = [
+            '/tmp/missing.cert',
+            '/tmp/good.cert']
+        await openstack_utils.async_block_until_ca_exists(
+            'keystone',
+            'CERTIFICATE')
+        self.assertTrue(self.result)
+
+        # Test a single missing
+        self._async_get_remote_ca_cert_file_candidates.return_value = [
+            '/tmp/missing.cert']
+        await openstack_utils.async_block_until_ca_exists(
+            'keystone',
+            'CERTIFICATE')
+        self.assertFalse(self.result)
+
+    async def test__async_get_remote_ca_cert_file_candidates(self):
+        self.patch_object(openstack_utils.zaza.model, "async_get_relation_id")
+        rel_id_out = {
+        }
+
+        def _get_relation_id(app, cert_app, model_name, remote_interface_name):
+            return rel_id_out[cert_app]
+        self.async_get_relation_id.side_effect = _get_relation_id
+
+        rel_id_out['vault'] = 'certs:1'
+        r = await openstack_utils._async_get_remote_ca_cert_file_candidates(
+            'neutron-api', 'mymodel')
+        self.assertEqual(
+            r,
+            ['/usr/local/share/ca-certificates/vault_juju_ca_cert.crt',
+             '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'])
+
+        rel_id_out['vault'] = None
+        r = await openstack_utils._async_get_remote_ca_cert_file_candidates(
+            'neutron-api', 'mymodel')
+        self.assertEqual(
+            r,
+            ['/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'])

unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py

@@ -44,25 +44,37 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
         self.patch_object(
             openstack_upgrade.zaza.model,
             "get_application_config")
+        self.patch_object(
+            openstack_upgrade.zaza.model,
+            "block_until_all_units_idle")
+        self.patch_object(
+            openstack_upgrade,
+            "block_until_mysql_innodb_cluster_has_rw")
 
         def _get_application_config(app, model_name=None):
             app_config = {
-                'ceph-mon': {'verbose': True, 'source': 'old-src'},
-                'neutron-openvswitch': {'verbose': True},
-                'ntp': {'verbose': True},
-                'percona-cluster': {'verbose': True, 'source': 'old-src'},
+                'ceph-mon': {'verbose': {'value': True},
+                             'source': {'value': 'old-src'}},
+                'neutron-openvswitch': {'verbose': {'value': True}},
+                'ntp': {'verbose': {'value': True}},
+                'percona-cluster': {'verbose': {'value': True},
+                                    'source': {'value': 'old-src'}},
                 'cinder': {
-                    'verbose': True,
-                    'openstack-origin': 'old-src',
-                    'action-managed-upgrade': False},
+                    'verbose': {'value': True},
+                    'openstack-origin': {'value': 'old-src'},
+                    'action-managed-upgrade': {'value': False}},
                 'neutron-api': {
-                    'verbose': True,
-                    'openstack-origin': 'old-src',
-                    'action-managed-upgrade': False},
+                    'verbose': {'value': True},
+                    'openstack-origin': {'value': 'old-src'},
+                    'action-managed-upgrade': {'value': False}},
                 'nova-compute': {
-                    'verbose': True,
-                    'openstack-origin': 'old-src',
-                    'action-managed-upgrade': False},
+                    'verbose': {'value': True},
+                    'openstack-origin': {'value': 'old-src'},
+                    'action-managed-upgrade': {'value': False}},
+                'mysql-innodb-cluster': {
+                    'verbose': {'value': True},
+                    'source': {'value': 'old-src'},
+                    'action-managed-upgrade': {'value': True}},
             }
             return app_config[app]
         self.get_application_config.side_effect = _get_application_config
@@ -74,6 +86,10 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
                 'subordinate-to': 'nova-compute'},
             'ntp': {  # Filter as it has no source option
                 'charm': 'cs:ntp'},
+            'mysql-innodb-cluster': {
+                'charm': 'cs:mysql-innodb-cluster',
+                'units': {
+                    'mysql-innodb-cluster/0': {}}},
             'nova-compute': {
                 'charm': 'cs:nova-compute',
                 'units': {
@@ -115,7 +131,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
             model_name=None,
             raise_on_failure=True)
 
-    def test_action_upgrade_group(self):
+    def test_action_upgrade_apps(self):
         self.patch_object(openstack_upgrade, "pause_units")
         self.patch_object(openstack_upgrade, "action_unit_upgrade")
         self.patch_object(openstack_upgrade, "resume_units")
@@ -127,7 +143,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
             'nova-compute': [mock_nova_compute_0],
             'cinder': [mock_cinder_1]}
         self.get_units.side_effect = lambda app, model_name: units[app]
-        openstack_upgrade.action_upgrade_group(['nova-compute', 'cinder'])
+        openstack_upgrade.action_upgrade_apps(['nova-compute', 'cinder'])
         pause_calls = [
             mock.call(['cinder-hacluster/0'], model_name=None),
             mock.call(['nova-compute/0', 'cinder/1'], model_name=None)]
@@ -142,6 +158,30 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
             mock.call(['cinder-hacluster/0'], model_name=None)]
         self.resume_units.assert_has_calls(resume_calls, any_order=False)
 
+    def test_action_upgrade_apps_mysql_innodb_cluster(self):
+        """Verify that mysql-innodb-cluster is settled before complete."""
+        self.patch_object(openstack_upgrade, "pause_units")
+        self.patch_object(openstack_upgrade, "action_unit_upgrade")
+        self.patch_object(openstack_upgrade, "resume_units")
+        mock_mysql_innodb_cluster_0 = mock.MagicMock()
+        mock_mysql_innodb_cluster_0.entity_id = 'mysql-innodb-cluster/0'
+        units = {'mysql-innodb-cluster': [mock_mysql_innodb_cluster_0]}
+        self.get_units.side_effect = lambda app, model_name: units[app]
+        openstack_upgrade.action_upgrade_apps(['mysql-innodb-cluster'])
+        pause_calls = [
+            mock.call(['mysql-innodb-cluster/0'], model_name=None)]
+        self.pause_units.assert_has_calls(pause_calls, any_order=False)
+        action_unit_upgrade_calls = [
+            mock.call(['mysql-innodb-cluster/0'], model_name=None)]
+        self.action_unit_upgrade.assert_has_calls(
+            action_unit_upgrade_calls,
+            any_order=False)
+        resume_calls = [
+            mock.call(['mysql-innodb-cluster/0'], model_name=None)]
+        self.resume_units.assert_has_calls(resume_calls, any_order=False)
+        self.block_until_mysql_innodb_cluster_has_rw.assert_called_once_with(
+            None)
+
     def test_set_upgrade_application_config(self):
         openstack_upgrade.set_upgrade_application_config(
             ['neutron-api', 'cinder'],
@@ -177,17 +217,23 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
         self.assertFalse(
             openstack_upgrade.is_action_upgradable('percona-cluster'))
 
+    def test_is_already_upgraded(self):
+        self.assertTrue(
+            openstack_upgrade.is_already_upgraded('cinder', 'old-src'))
+        self.assertFalse(
+            openstack_upgrade.is_already_upgraded('cinder', 'new-src'))
+
     def test_run_action_upgrade(self):
         self.patch_object(openstack_upgrade, "set_upgrade_application_config")
-        self.patch_object(openstack_upgrade, "action_upgrade_group")
-        openstack_upgrade.run_action_upgrade(
+        self.patch_object(openstack_upgrade, "action_upgrade_apps")
+        openstack_upgrade.run_action_upgrades(
             ['cinder', 'neutron-api'],
             'new-src')
         self.set_upgrade_application_config.assert_called_once_with(
             ['cinder', 'neutron-api'],
             'new-src',
             model_name=None)
-        self.action_upgrade_group.assert_called_once_with(
+        self.action_upgrade_apps.assert_called_once_with(
             ['cinder', 'neutron-api'],
             model_name=None)
 
@@ -196,7 +242,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
         self.patch_object(
             openstack_upgrade.zaza.model,
             'block_until_all_units_idle')
-        openstack_upgrade.run_all_in_one_upgrade(
+        openstack_upgrade.run_all_in_one_upgrades(
             ['percona-cluster'],
             'new-src')
         self.set_upgrade_application_config.assert_called_once_with(
@@ -207,34 +253,36 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
         self.block_until_all_units_idle.assert_called_once_with()
 
     def test_run_upgrade(self):
-        self.patch_object(openstack_upgrade, "run_all_in_one_upgrade")
-        self.patch_object(openstack_upgrade, "run_action_upgrade")
-        openstack_upgrade.run_upgrade(
+        self.patch_object(openstack_upgrade, "run_all_in_one_upgrades")
+        self.patch_object(openstack_upgrade, "run_action_upgrades")
+        openstack_upgrade.run_upgrade_on_apps(
             ['cinder', 'neutron-api', 'ceph-mon'],
             'new-src')
-        self.run_all_in_one_upgrade.assert_called_once_with(
+        self.run_all_in_one_upgrades.assert_called_once_with(
             ['ceph-mon'],
             'new-src',
             model_name=None)
-        self.run_action_upgrade.assert_called_once_with(
+        self.run_action_upgrades.assert_called_once_with(
             ['cinder', 'neutron-api'],
             'new-src',
             model_name=None)
 
     def test_run_upgrade_tests(self):
-        self.patch_object(openstack_upgrade, "run_upgrade")
+        self.patch_object(openstack_upgrade, "run_upgrade_on_apps")
         self.patch_object(openstack_upgrade, "get_upgrade_groups")
-        self.get_upgrade_groups.return_value = {
-            'Compute': ['nova-compute'],
-            'Control Plane': ['cinder', 'neutron-api'],
-            'Core Identity': ['keystone'],
-            'Storage': ['ceph-mon'],
-            'sweep_up': ['designate']}
+        self.get_upgrade_groups.return_value = [
+            ('Compute', ['nova-compute']),
+            ('Control Plane', ['cinder', 'neutron-api']),
+            ('Core Identity', ['keystone']),
+            ('Storage', ['ceph-mon']),
+            ('sweep_up', ['designate'])]
         openstack_upgrade.run_upgrade_tests('new-src', model_name=None)
         run_upgrade_calls = [
+            mock.call(['nova-compute'], 'new-src', model_name=None),
+            mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None),
             mock.call(['keystone'], 'new-src', model_name=None),
             mock.call(['ceph-mon'], 'new-src', model_name=None),
-            mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None),
-            mock.call(['nova-compute'], 'new-src', model_name=None),
-            mock.call(['designate'], 'new-src', model_name=None)]
-        self.run_upgrade.assert_has_calls(run_upgrade_calls, any_order=False)
+            mock.call(['designate'], 'new-src', model_name=None),
+        ]
+        self.run_upgrade_on_apps.assert_has_calls(
+            run_upgrade_calls, any_order=False)

unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import asyncio
 import mock
 import sys
 import unittest
@@ -139,28 +138,7 @@ class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase):
         self.assertEqual(expected, config)
 
 
-class AioTestCase(ut_utils.BaseTestCase):
-    def __init__(self, methodName='runTest', loop=None):
-        self.loop = loop or asyncio.get_event_loop()
-        self._function_cache = {}
-        super(AioTestCase, self).__init__(methodName=methodName)
-
-    def coroutine_function_decorator(self, func):
-        def wrapper(*args, **kw):
-            return self.loop.run_until_complete(func(*args, **kw))
-        return wrapper
-
-    def __getattribute__(self, item):
-        attr = object.__getattribute__(self, item)
-        if asyncio.iscoroutinefunction(attr) and item.startswith('test_'):
-            if item not in self._function_cache:
-                self._function_cache[item] = (
-                    self.coroutine_function_decorator(attr))
-            return self._function_cache[item]
-        return attr
-
-
-class TestParallelSeriesUpgrade(AioTestCase):
+class TestParallelSeriesUpgrade(ut_utils.AioTestCase):
     def setUp(self):
         super(TestParallelSeriesUpgrade, self).setUp()
         if sys.version_info < (3, 6, 0):
@@ -182,6 +160,8 @@ class TestParallelSeriesUpgrade(AioTestCase):
         self.model.async_wait_for_unit_idle = mock.AsyncMock()
         self.async_run_on_machine = mock.AsyncMock()
         self.model.async_run_on_machine = self.async_run_on_machine
+        self.model.async_block_until_units_on_machine_are_idle = \
+            mock.AsyncMock()
 
     @mock.patch.object(upgrade_utils.cl_utils, 'get_class')
     async def test_run_post_application_upgrade_functions(
@@ -492,7 +472,13 @@ class TestParallelSeriesUpgrade(AioTestCase):
         mock_remove_confdef_file.assert_called_once_with('1')
         mock_add_confdef_file.assert_called_once_with('1')
 
-    async def test_maybe_pause_things_primary(self):
+    @mock.patch("asyncio.gather")
+    async def test_maybe_pause_things_primary(self, mock_gather):
+        async def _gather(*args):
+            for f in args:
+                await f
+
+        mock_gather.side_effect = _gather
         await upgrade_utils.maybe_pause_things(
             FAKE_STATUS,
             ['app/1', 'app/2'],
@@ -503,7 +489,13 @@ class TestParallelSeriesUpgrade(AioTestCase):
             mock.call('app/2', "pause", action_params={}),
         ])
 
-    async def test_maybe_pause_things_subordinates(self):
+    @mock.patch("asyncio.gather")
+    async def test_maybe_pause_things_subordinates(self, mock_gather):
+        async def _gather(*args):
+            for f in args:
+                await f
+
+        mock_gather.side_effect = _gather
         await upgrade_utils.maybe_pause_things(
             FAKE_STATUS,
             ['app/1', 'app/2'],
@@ -514,7 +506,13 @@ class TestParallelSeriesUpgrade(AioTestCase):
             mock.call('app-hacluster/2', "pause", action_params={}),
         ])
 
-    async def test_maybe_pause_things_all(self):
+    @mock.patch("asyncio.gather")
+    async def test_maybe_pause_things_all(self, mock_gather):
+        async def _gather(*args):
+            for f in args:
+                await f
+
+        mock_gather.side_effect = _gather
         await upgrade_utils.maybe_pause_things(
             FAKE_STATUS,
             ['app/1', 'app/2'],

unit_tests/utils.py

@@ -19,6 +19,7 @@
 
 """Module to provide helper for writing unit tests."""
 
+import asyncio
 import contextlib
 import io
 import mock
@@ -96,3 +97,24 @@ class BaseTestCase(unittest.TestCase):
             started.return_value = return_value
         self._patches_start[name] = started
         setattr(self, name, started)
+
+
+class AioTestCase(BaseTestCase):
+    def __init__(self, methodName='runTest', loop=None):
+        self.loop = loop or asyncio.get_event_loop()
+        self._function_cache = {}
+        super(AioTestCase, self).__init__(methodName=methodName)
+
+    def coroutine_function_decorator(self, func):
+        def wrapper(*args, **kw):
+            return self.loop.run_until_complete(func(*args, **kw))
+        return wrapper
+
+    def __getattribute__(self, item):
+        attr = object.__getattribute__(self, item)
+        if asyncio.iscoroutinefunction(attr) and item.startswith('test_'):
+            if item not in self._function_cache:
+                self._function_cache[item] = (
+                    self.coroutine_function_decorator(attr))
+            return self._function_cache[item]
+        return attr

zaza/openstack/charm_tests/aodh/tests.py

@@ -25,6 +25,7 @@ import zaza.model
 import zaza.openstack.configure.guest
 import zaza.openstack.charm_tests.glance.setup as glance_setup
 import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.openstack.utilities.generic as generic_utils
 import zaza.openstack.utilities.openstack as openstack_utils
 import zaza.openstack.configure.telemetry as telemetry_utils
 
@@ -70,6 +71,13 @@ class AodhTest(test_utils.OpenStackBaseTest):
         """Check that aodh api is responding."""
         self.aodh_client.alarm.list()
 
+    @tenacity.retry(
+        retry=tenacity.retry_if_result(lambda ret: ret is not None),
+        wait=tenacity.wait_fixed(120),
+        stop=tenacity.stop_after_attempt(2))
+    def _retry_check_commands_on_units(self, cmds, units):
+        return generic_utils.check_commands_on_units(cmds, units)
+
     @property
     def services(self):
         """Return a list of the service that should be running."""
@@ -139,6 +147,26 @@ class AodhTest(test_utils.OpenStackBaseTest):
             logging.info("Testing pause resume")
         self.query_aodh_api()
 
+    def test_902_nrpe_service_checks(self):
+        """Confirm that the NRPE service check files are created."""
+        units = zaza.model.get_units('aodh')
+        cmds = []
+        if self.release >= self.xenial_ocata:
+            services = ['aodh-evaluator', 'aodh-notifier',
+                        'aodh-listener', 'apache2']
+        else:
+            services = ['aodh-api', 'aodh-evaluator',
+                        'aodh-notifier', 'aodh-listener']
+        for check_name in services:
+            cmds.append(
+                'egrep -oh /usr/local.* /etc/nagios/nrpe.d/'
+                'check_{}.cfg'.format(check_name)
+            )
+        ret = self._retry_check_commands_on_units(cmds, units)
+        if ret:
+            logging.info(ret)
+        self.assertIsNone(ret, msg=ret)
+
 
 class AodhServerAlarmTest(test_utils.OpenStackBaseTest):
     """Test server events trigger Aodh alarms."""

zaza/openstack/charm_tests/ceph/dashboard/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Collection of code for setting up and testing ``ceph-dashboard``."""

zaza/openstack/charm_tests/ceph/dashboard/tests.py

@@ -0,0 +1,97 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Encapsulating `ceph-dashboard` testing."""
+
+import collections
+import os
+import requests
+
+import zaza
+import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.utilities.deployment_env as deployment_env
+
+
+class CephDashboardTest(test_utils.BaseCharmTest):
+    """Class for `ceph-dashboard` tests."""
+
+    REMOTE_CERT_FILE = ('/usr/local/share/ca-certificates/'
+                        'vault_ca_cert_dashboard.crt')
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running ceph dashboard tests."""
+        super().setUpClass()
+        cls.application_name = 'ceph-dashboard'
+        cls.local_ca_cert = cls.collect_ca()
+
+    @classmethod
+    def collect_ca(cls):
+        """Collect CA from ceph-dashboard unit."""
+        local_ca_cert = os.path.join(
+            deployment_env.get_tmpdir(),
+            os.path.basename(cls.REMOTE_CERT_FILE))
+        if not os.path.isfile(local_ca_cert):
+            units = zaza.model.get_units(cls.application_name)
+            zaza.model.scp_from_unit(
+                units[0].entity_id,
+                cls.REMOTE_CERT_FILE,
+                local_ca_cert)
+        return local_ca_cert
+
+    def test_dashboard_units(self):
+        """Check dashboard units are configured correctly."""
+        # XXX: Switch to using CA for verification when
+        #      https://bugs.launchpad.net/cloud-archive/+bug/1933410
+        #      is fix released.
+        # verify = self.local_ca_cert
+        verify = False
+        units = zaza.model.get_units(self.application_name)
+        rcs = collections.defaultdict(list)
+        for unit in units:
+            r = requests.get(
+                'https://{}:8443'.format(unit.public_address),
+                verify=verify,
+                allow_redirects=False)
+            rcs[r.status_code].append(unit.public_address)
+        self.assertEqual(len(rcs[requests.codes.ok]), 1)
+        self.assertEqual(len(rcs[requests.codes.see_other]), len(units) - 1)
+
+    def create_user(self, username, role='administrator'):
+        """Create a dashboard user.
+
+        :param username: Username to create.
+        :type username: str
+        :param role: Role to grant to user.
+        :type role: str
+        :returns: Results from action.
+        :rtype: juju.action.Action
+        """
+        action = zaza.model.run_action_on_leader(
+            'ceph-dashboard',
+            'add-user',
+            action_params={
+                'username': username,
+                'role': role})
+        return action
+
+    def test_create_user(self):
+        """Test create user action."""
+        test_user = 'marvin'
+        action = self.create_user(test_user)
+        self.assertEqual(action.status, "completed")
+        self.assertTrue(action.data['results']['password'])
+        action = self.create_user(test_user)
+        # Action should fail as the user already exists
+        self.assertEqual(action.status, "failed")

zaza/openstack/charm_tests/ceph/iscsi/tests.py

@@ -64,26 +64,25 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
         :rtype: Dict
         """
         gw_units = zaza.model.get_units('ceph-iscsi')
-        primary_gw = gw_units[0]
-        secondary_gw = gw_units[1]
         host_names = generic_utils.get_unit_hostnames(gw_units, fqdn=True)
         client_entity_ids = [
             u.entity_id for u in zaza.model.get_units('ubuntu')]
         ctxt = {
             'client_entity_ids': sorted(client_entity_ids),
             'gw_iqn': self.GW_IQN,
-            'gw1_ip': primary_gw.public_address,
-            'gw1_hostname': host_names[primary_gw.entity_id],
-            'gw1_entity_id': primary_gw.entity_id,
-            'gw2_ip': secondary_gw.public_address,
-            'gw2_hostname': host_names[secondary_gw.entity_id],
-            'gw2_entity_id': secondary_gw.entity_id,
             'chap_creds': 'username={chap_username} password={chap_password}',
             'gwcli_gw_dir': '/iscsi-targets/{gw_iqn}/gateways',
             'gwcli_hosts_dir': '/iscsi-targets/{gw_iqn}/hosts',
             'gwcli_disk_dir': '/disks',
             'gwcli_client_dir': '{gwcli_hosts_dir}/{client_initiatorname}',
         }
+        ctxt['gateway_units'] = [
+            {
+                'entity_id': u.entity_id,
+                'ip': u.public_address,
+                'hostname': host_names[u.entity_id]}
+            for u in zaza.model.get_units('ceph-iscsi')]
+        ctxt['gw_ip'] = sorted([g['ip'] for g in ctxt['gateway_units']])[0]
         return ctxt
 
     def run_commands(self, unit_name, commands, ctxt):
@@ -116,9 +115,8 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
             'ceph-iscsi',
             'create-target',
             action_params={
-                'gateway-units': '{} {}'.format(
-                    ctxt['gw1_entity_id'],
-                    ctxt['gw2_entity_id']),
+                'gateway-units': ' '.join([g['entity_id']
+                                           for g in ctxt['gateway_units']]),
                 'iqn': self.GW_IQN,
                 'rbd-pool-name': ctxt.get('pool_name', ''),
                 'ec-rbd-metadata-pool': ctxt.get('ec_meta_pool_name', ''),
@@ -139,7 +137,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest):
         base_op_cmd = ('iscsiadm --mode node --targetname {gw_iqn} '
                        '--op=update ').format(**ctxt)
         setup_cmds = [
-            'iscsiadm -m discovery -t st -p {gw1_ip}',
+            'iscsiadm -m discovery -t st -p {gw_ip}',
             base_op_cmd + '-n node.session.auth.authmethod -v CHAP',
             base_op_cmd + '-n node.session.auth.username -v {chap_username}',
             base_op_cmd + '-n node.session.auth.password -v {chap_password}',

zaza/openstack/charm_tests/ceph/osd/tests.py

@@ -16,6 +16,9 @@
 
 import logging
 import unittest
+import re
+
+from copy import deepcopy
 
 import zaza.openstack.charm_tests.test_utils as test_utils
 import zaza.model as zaza_model
@@ -47,3 +50,235 @@ class SecurityTest(unittest.TestCase):
             expected_passes,
             expected_failures,
             expected_to_pass=True)
+
+
+class OsdService:
+    """Simple representation of ceph-osd systemd service."""
+
+    def __init__(self, id_):
+        """
+        Init service using its ID.
+
+        e.g.: id_=1 -> ceph-osd@1
+        """
+        self.id = id_
+        self.name = 'ceph-osd@{}'.format(id_)
+
+
+async def async_wait_for_service_status(unit_name, services, target_status,
+                                        model_name=None, timeout=2700):
+    """Wait for all services on the unit to be in the desired state.
+
+    Note: This function emulates the
+    `zaza.model.async_block_until_service_status` function, but it's using
+    `systemctl is-active` command instead of `pidof/pgrep` of the original
+    function.
+
+    :param unit_name: Name of unit to run action on
+    :type unit_name: str
+    :param services: List of services to check
+    :type services: List[str]
+    :param target_status: State services must be in (stopped or running)
+    :type target_status: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :param timeout: Time to wait for status to be achieved
+    :type timeout: int
+    """
+    async def _check_service():
+        services_ok = True
+        for service in services:
+            command = r"systemctl is-active '{}'".format(service)
+            out = await zaza_model.async_run_on_unit(
+                unit_name,
+                command,
+                model_name=model_name,
+                timeout=timeout)
+            response = out['Stdout'].strip()
+
+            if target_status == "running" and response == 'active':
+                continue
+            elif target_status == "stopped" and response == 'inactive':
+                continue
+            else:
+                services_ok = False
+                break
+
+        return services_ok
+
+    accepted_states = ('stopped', 'running')
+    if target_status not in accepted_states:
+        raise RuntimeError('Invalid target state "{}". Accepted states: '
+                           '{}'.format(target_status, accepted_states))
+
+    async with zaza_model.run_in_model(model_name):
+        await zaza_model.async_block_until(_check_service, timeout=timeout)
+
+
+wait_for_service = zaza_model.sync_wrapper(async_wait_for_service_status)
+
+
+class ServiceTest(unittest.TestCase):
+    """ceph-osd systemd service tests."""
+
+    TESTED_UNIT = 'ceph-osd/0'  # This can be any ceph-osd unit in the model
+    SERVICE_PATTERN = re.compile(r'ceph-osd@(?P<service_id>\d+)\.service')
+
+    def __init__(self, methodName='runTest'):
+        """Initialize Test Case."""
+        super(ServiceTest, self).__init__(methodName)
+        self._available_services = None
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running ceph service tests."""
+        super(ServiceTest, cls).setUpClass()
+
+    def setUp(self):
+        """Run test setup."""
+        # Skip 'service' action tests on systems without systemd
+        result = zaza_model.run_on_unit(self.TESTED_UNIT, 'which systemctl')
+        if not result['Stdout']:
+            raise unittest.SkipTest("'service' action is not supported on "
+                                    "systems without 'systemd'. Skipping "
+                                    "tests.")
+        # Note(mkalcok): This counter reset is needed because ceph-osd service
+        #       is limited to 3 restarts per 30 mins which is insufficient
+        #       when running functional tests for 'service' action. This
+        #       limitation is defined in /lib/systemd/system/ceph-osd@.service
+        #       in section [Service] with options 'StartLimitInterval' and
+        #       'StartLimitBurst'
+        reset_counter = 'systemctl reset-failed'
+        zaza_model.run_on_unit(self.TESTED_UNIT, reset_counter)
+
+    def tearDown(self):
+        """Start ceph-osd services after each test.
+
+        This ensures that the environment is ready for the next tests.
+        """
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start',
+                                       action_params={'osds': 'all'},
+                                       raise_on_failure=True)
+
+    @property
+    def available_services(self):
+        """Return list of all ceph-osd services present on the TESTED_UNIT."""
+        if self._available_services is None:
+            self._available_services = self._fetch_osd_services()
+        return self._available_services
+
+    def _fetch_osd_services(self):
+        """Fetch all ceph-osd services present on the TESTED_UNIT."""
+        service_list = []
+        service_list_cmd = 'systemctl list-units --full --all ' \
+                           '--no-pager -t service'
+        result = zaza_model.run_on_unit(self.TESTED_UNIT, service_list_cmd)
+        for line in result['Stdout'].split('\n'):
+            service_name = self.SERVICE_PATTERN.search(line)
+            if service_name:
+                service_id = int(service_name.group('service_id'))
+                service_list.append(OsdService(service_id))
+        return service_list
+
+    def test_start_stop_all_by_keyword(self):
+        """Start and Stop all ceph-osd services using keyword 'all'."""
+        service_list = [service.name for service in self.available_services]
+
+        logging.info("Running 'service stop=all' action on {} "
+                     "unit".format(self.TESTED_UNIT))
+        zaza_model.run_action_on_units([self.TESTED_UNIT], 'stop',
+                                       action_params={'osds': 'all'})
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_list,
+                         target_status='stopped')
+
+        logging.info("Running 'service start=all' action on {} "
+                     "unit".format(self.TESTED_UNIT))
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start',
+                                       action_params={'osds': 'all'})
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_list,
+                         target_status='running')
+
+    def test_start_stop_all_by_list(self):
+        """Start and Stop all ceph-osd services using explicit list."""
+        service_list = [service.name for service in self.available_services]
+        service_ids = [str(service.id) for service in self.available_services]
+        action_params = ','.join(service_ids)
+
+        logging.info("Running 'service stop={}' action on {} "
+                     "unit".format(action_params, self.TESTED_UNIT))
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop',
+                                       action_params={'osds': action_params})
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_list,
+                         target_status='stopped')
+
+        logging.info("Running 'service start={}' action on {} "
+                     "unit".format(action_params, self.TESTED_UNIT))
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start',
+                                       action_params={'osds': action_params})
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_list,
+                         target_status='running')
+
+    def test_stop_specific(self):
+        """Stop only specified ceph-osd service."""
+        if len(self.available_services) < 2:
+            raise unittest.SkipTest('This test can be performed only if '
+                                    'there\'s more than one ceph-osd service '
+                                    'present on the tested unit')
+
+        should_run = deepcopy(self.available_services)
+        to_stop = should_run.pop()
+        should_run = [service.name for service in should_run]
+
+        logging.info("Running 'service stop={} on {} "
+                     "unit".format(to_stop.id, self.TESTED_UNIT))
+
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop',
+                                       action_params={'osds': to_stop.id})
+
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=[to_stop.name, ],
+                         target_status='stopped')
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=should_run,
+                         target_status='running')
+
+    def test_start_specific(self):
+        """Start only specified ceph-osd service."""
+        if len(self.available_services) < 2:
+            raise unittest.SkipTest('This test can be performed only if '
+                                    'there\'s more than one ceph-osd service '
+                                    'present on the tested unit')
+
+        service_names = [service.name for service in self.available_services]
+        should_stop = deepcopy(self.available_services)
+        to_start = should_stop.pop()
+        should_stop = [service.name for service in should_stop]
+
+        # Note: can't stop ceph-osd.target as restarting a single OSD will
+        # cause this to start all of the OSDs when a single one starts.
+        logging.info("Stopping all running ceph-osd services")
+        service_stop_cmd = '; '.join(['systemctl stop {}'.format(service)
+                                      for service in service_names])
+        zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd)
+
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=service_names,
+                         target_status='stopped')
+
+        logging.info("Running 'service start={} on {} "
+                     "unit".format(to_start.id, self.TESTED_UNIT))
+
+        zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start',
+                                       action_params={'osds': to_start.id})
+
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=[to_start.name, ],
+                         target_status='running')
+
+        wait_for_service(unit_name=self.TESTED_UNIT,
+                         services=should_stop,
+                         target_status='stopped')

zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py

@@ -17,6 +17,8 @@ import json
 import logging
 import re
 
+import cinderclient.exceptions as cinder_exceptions
+
 import zaza.openstack.charm_tests.test_utils as test_utils
 
 import zaza.model
@@ -28,6 +30,129 @@ from zaza.openstack.charm_tests.glance.setup import (
     CIRROS_IMAGE_NAME)
 
 
+DEFAULT_CINDER_RBD_MIRRORING_MODE = 'pool'
+
+
+def get_cinder_rbd_mirroring_mode(cinder_ceph_app_name='cinder-ceph'):
+    """Get the RBD mirroring mode for the Cinder Ceph pool.
+
+    :param cinder_ceph_app_name: Cinder Ceph Juju application name.
+    :type cinder_ceph_app_name: str
+    :returns: A string representing the RBD mirroring mode. It can be
+              either 'pool' or 'image'.
+    :rtype: str
+    """
+    rbd_mirroring_mode_config = zaza.model.get_application_config(
+        cinder_ceph_app_name).get('rbd-mirroring-mode')
+    if rbd_mirroring_mode_config:
+        rbd_mirroring_mode = rbd_mirroring_mode_config.get(
+            'value', DEFAULT_CINDER_RBD_MIRRORING_MODE).lower()
+    else:
+        rbd_mirroring_mode = DEFAULT_CINDER_RBD_MIRRORING_MODE
+
+    return rbd_mirroring_mode
+
+
+def get_glance_image(glance):
+    """Get the Glance image object to be used by the Ceph tests.
+
+    It looks for the Cirros Glance image, and it's returned if it's found.
+    If the Cirros image is not found, it will try and find the Ubuntu
+    LTS image.
+
+    :param glance: Authenticated glanceclient
+    :type glance: glanceclient.Client
+    :returns: Glance image object
+    :rtype: glanceclient.image
+    """
+    images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME)
+    if images:
+        return images[0]
+    logging.info("Failed to find {} image, falling back to {}".format(
+        CIRROS_IMAGE_NAME,
+        LTS_IMAGE_NAME))
+    return openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0]
+
+
+def setup_cinder_repl_volume_type(cinder, type_name='repl',
+                                  backend_name='cinder-ceph'):
+    """Set up the Cinder volume replication type.
+
+    :param cinder: Authenticated cinderclient
+    :type cinder: cinder.Client
+    :param type_name: Cinder volume type name
+    :type type_name: str
+    :param backend_name: Cinder volume backend name with replication enabled.
+    :type backend_name: str
+    :returns: Cinder volume type object
+    :rtype: cinderclient.VolumeType
+    """
+    try:
+        vol_type = cinder.volume_types.find(name=type_name)
+    except cinder_exceptions.NotFound:
+        vol_type = cinder.volume_types.create(type_name)
+
+    vol_type.set_keys(metadata={
+        'volume_backend_name': backend_name,
+        'replication_enabled': '<is> True',
+    })
+    return vol_type
+
+
+# TODO: This function should be incorporated into
+# 'zaza.openstack.utilities.openstack.create_volume' helper, once the below
+# flakiness comments are addressed.
+def create_cinder_volume(cinder, name='zaza', image_id=None, type_id=None):
+    """Create a new Cinder volume.
+
+    :param cinder: Authenticated cinderclient.
+    :type cinder: cinder.Client
+    :param name: Volume name.
+    :type name: str
+    :param image_id: Glance image id, if the volume is created from image.
+    :type image_id: str
+    :param type_id: Cinder Volume type id, if the volume needs to use an
+                    explicit volume type.
+    :type type_id: boolean
+    :returns: Cinder volume
+    :rtype: :class:`Volume`.
+    """
+    # NOTE(fnordahl): for some reason create volume from image often fails
+    # when run just after deployment is finished.  We should figure out
+    # why, resolve the underlying issue and then remove this.
+    #
+    # We do not use tenacity here as it will interfere with tenacity used
+    # in ``resource_reaches_status``
+    def create_volume(cinder, volume_params, retry=20):
+        if retry < 1:
+            return
+        volume = cinder.volumes.create(**volume_params)
+        try:
+            # Note(coreycb): stop_after_attempt is increased because using
+            # juju storage for ceph-osd backed by cinder on undercloud
+            # takes longer than the prior method of directory-backed OSD
+            # devices.
+            openstack.resource_reaches_status(
+                cinder.volumes, volume.id, msg='volume',
+                stop_after_attempt=20)
+            return volume
+        except AssertionError:
+            logging.info('retrying')
+            volume.delete()
+            return create_volume(cinder, volume_params, retry=retry - 1)
+
+    volume_params = {
+        'size': 8,
+        'name': name,
+    }
+    if image_id:
+        volume_params['imageRef'] = image_id
+    if type_id:
+        volume_params['volume_type'] = type_id
+
+    return create_volume(cinder, volume_params)
+
+
 class CephRBDMirrorBase(test_utils.OpenStackBaseTest):
     """Base class for ``ceph-rbd-mirror`` tests."""
 
@@ -35,20 +160,26 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest):
     def setUpClass(cls):
         """Run setup for ``ceph-rbd-mirror`` tests."""
         super().setUpClass()
+        cls.cinder_ceph_app_name = 'cinder-ceph'
+        cls.test_cinder_volume_name = 'test-cinder-ceph-volume'
         # get ready for multi-model Zaza
         cls.site_a_model = cls.site_b_model = zaza.model.get_juju_model()
         cls.site_b_app_suffix = '-b'
 
-    def run_status_action(self, application_name=None, model_name=None):
+    def run_status_action(self, application_name=None, model_name=None,
+                          pools=[]):
         """Run status action, decode and return response."""
+        action_params = {
+            'verbose': True,
+            'format': 'json',
+        }
+        if len(pools) > 0:
+            action_params['pools'] = ','.join(pools)
         result = zaza.model.run_action_on_leader(
             application_name or self.application_name,
             'status',
             model_name=model_name,
-            action_params={
-                'verbose': True,
-                'format': 'json',
-            })
+            action_params=action_params)
         return json.loads(result.results['output'])
 
     def get_pools(self):
@@ -68,10 +199,26 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest):
             model_name=self.site_b_model)
         return sorted(site_a_pools.keys()), sorted(site_b_pools.keys())
 
+    def get_failover_pools(self):
+        """Get the failover Ceph pools' names, from both sites.
+
+        If the Cinder RBD mirroring mode is 'image', the 'cinder-ceph' pool
+        needs to be excluded, since Cinder orchestrates the failover then.
+
+        :returns: Tuple with site-a pools and site-b pools.
+        :rtype: Tuple[List[str], List[str]]
+        """
+        site_a_pools, site_b_pools = self.get_pools()
+        if get_cinder_rbd_mirroring_mode(self.cinder_ceph_app_name) == 'image':
+            site_a_pools.remove(self.cinder_ceph_app_name)
+            site_b_pools.remove(self.cinder_ceph_app_name)
+        return site_a_pools, site_b_pools
+
     def wait_for_mirror_state(self, state, application_name=None,
                               model_name=None,
                               check_entries_behind_master=False,
-                              require_images_in=[]):
+                              require_images_in=[],
+                              pools=[]):
         """Wait until all images reach requested state.
 
         This function runs the ``status`` action and examines the data it
@@ -90,6 +237,9 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest):
         :type check_entries_behind_master: bool
         :param require_images_in: List of pools to require images in
         :type require_images_in: list of str
+        :param pools: List of pools to run status on. If this is empty, the
+                      status action will run on all the pools.
+        :type pools: list of str
         :returns: True on success, never returns on failure
         """
         rep = re.compile(r'.*entries_behind_master=(\d+)')
@@ -97,7 +247,8 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest):
             try:
                 # encapsulate in try except to work around LP: #1820976
                 pool_status = self.run_status_action(
-                    application_name=application_name, model_name=model_name)
+                    application_name=application_name, model_name=model_name,
+                    pools=pools)
             except KeyError:
                 continue
             for pool, status in pool_status.items():
@@ -124,6 +275,41 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest):
                 # all images with state has expected state
                 return True
 
+    def setup_test_cinder_volume(self):
+        """Set up the test Cinder volume into the Ceph RBD mirror environment.
+
+        If the volume already exists, then it's returned.
+
+        Also, if the Cinder RBD mirroring mode is 'image', the volume will
+        use an explicit volume type with the appropriate replication flags.
+        Otherwise, it is just a simple Cinder volume using the default backend.
+
+        :returns: Cinder volume
+        :rtype: :class:`Volume`.
+        """
+        session = openstack.get_overcloud_keystone_session()
+        cinder = openstack.get_cinder_session_client(session, version=3)
+
+        try:
+            return cinder.volumes.find(name=self.test_cinder_volume_name)
+        except cinder_exceptions.NotFound:
+            logging.info("Test Cinder volume doesn't exist. Creating it")
+
+        glance = openstack.get_glance_session_client(session)
+        image = get_glance_image(glance)
+        kwargs = {
+            'cinder': cinder,
+            'name': self.test_cinder_volume_name,
+            'image_id': image.id,
+        }
+        if get_cinder_rbd_mirroring_mode(self.cinder_ceph_app_name) == 'image':
+            volume_type = setup_cinder_repl_volume_type(
+                cinder,
+                backend_name=self.cinder_ceph_app_name)
+            kwargs['type_id'] = volume_type.id
+
+        return create_cinder_volume(**kwargs)
+
 
 class CephRBDMirrorTest(CephRBDMirrorBase):
     """Encapsulate ``ceph-rbd-mirror`` tests."""
@@ -195,44 +381,7 @@ class CephRBDMirrorTest(CephRBDMirrorBase):
         site B and subsequently comparing the contents we get a full end to end
         test.
         """
-        session = openstack.get_overcloud_keystone_session()
-        glance = openstack.get_glance_session_client(session)
-        cinder = openstack.get_cinder_session_client(session)
-
-        images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME)
-        if images:
-            image = images[0]
-        else:
-            logging.info("Failed to find {} image, falling back to {}".format(
-                CIRROS_IMAGE_NAME,
-                LTS_IMAGE_NAME))
-            image = openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0]
-
-        # NOTE(fnordahl): for some reason create volume from image often fails
-        # when run just after deployment is finished.  We should figure out
-        # why, resolve the underlying issue and then remove this.
-        #
-        # We do not use tenacity here as it will interfere with tenacity used
-        # in ``resource_reaches_status``
-        def create_volume_from_image(cinder, image, retry=20):
-            if retry < 1:
-                return
-            volume = cinder.volumes.create(8, name='zaza', imageRef=image.id)
-            try:
-                # Note(coreycb): stop_after_attempt is increased because using
-                # juju storage for ceph-osd backed by cinder on undercloud
-                # takes longer than the prior method of directory-backed OSD
-                # devices.
-                openstack.resource_reaches_status(
-                    cinder.volumes, volume.id, msg='volume',
-                    stop_after_attempt=20)
-                return volume
-            except AssertionError:
-                logging.info('retrying')
-                volume.delete()
-                return create_volume_from_image(cinder, image, retry=retry - 1)
-        volume = create_volume_from_image(cinder, image)
-
+        volume = self.setup_test_cinder_volume()
         site_a_hash = zaza.openstack.utilities.ceph.get_rbd_hash(
             zaza.model.get_lead_unit_name('ceph-mon',
                                           model_name=self.site_a_model),
@@ -244,6 +393,8 @@ class CephRBDMirrorTest(CephRBDMirrorBase):
             check_entries_behind_master=True,
             application_name=self.application_name + self.site_b_app_suffix,
             model_name=self.site_b_model)
+        logging.info('Checking the Ceph RBD hashes of the primary and '
+                     'the secondary Ceph images')
         site_b_hash = zaza.openstack.utilities.ceph.get_rbd_hash(
             zaza.model.get_lead_unit_name('ceph-mon' + self.site_b_app_suffix,
                                           model_name=self.site_b_model),
@@ -258,102 +409,399 @@ class CephRBDMirrorTest(CephRBDMirrorBase):
 class CephRBDMirrorControlledFailoverTest(CephRBDMirrorBase):
     """Encapsulate ``ceph-rbd-mirror`` controlled failover tests."""
 
-    def test_fail_over_fall_back(self):
-        """Validate controlled fail over and fall back."""
-        site_a_pools, site_b_pools = self.get_pools()
+    def execute_failover_juju_actions(self,
+                                      primary_site_app_name,
+                                      primary_site_model,
+                                      primary_site_pools,
+                                      secondary_site_app_name,
+                                      secondary_site_model,
+                                      secondary_site_pools):
+        """Execute the failover Juju actions.
+
+        The failover / failback via Juju actions shares the same workflow. The
+        failback is just a failover with sites in reversed order.
+
+        This function encapsulates the tasks to failover a primary site to
+        a secondary site:
+        1. Demote primary site
+        2. Validation of the primary site demotion
+        3. Promote secondary site
+        4. Validation of the secondary site promotion
+
+        :param primary_site_app_name: Primary site Ceph RBD mirror app name.
+        :type primary_site_app_name: str
+        :param primary_site_model: Primary site Juju model name.
+        :type primary_site_model: str
+        :param primary_site_pools: Primary site pools.
+        :type primary_site_pools: List[str]
+        :param secondary_site_app_name: Secondary site Ceph RBD mirror
+                                        app name.
+        :type secondary_site_app_name: str
+        :param secondary_site_model: Secondary site Juju model name.
+        :type secondary_site_model: str
+        :param secondary_site_pools: Secondary site pools.
+        :type secondary_site_pools: List[str]
+        """
+        # Check if primary and secondary pools sizes are the same.
+        self.assertEqual(len(primary_site_pools), len(secondary_site_pools))
+
+        # Run the 'demote' Juju action against the primary site pools.
+        logging.info('Demoting {} from model {}.'.format(
+            primary_site_app_name, primary_site_model))
         result = zaza.model.run_action_on_leader(
-            'ceph-rbd-mirror',
+            primary_site_app_name,
             'demote',
-            model_name=self.site_a_model,
-            action_params={})
+            model_name=primary_site_model,
+            action_params={
+                'pools': ','.join(primary_site_pools)
+            })
         logging.info(result.results)
+        self.assertEqual(int(result.results['Code']), 0)
+
+        # Validate that the demoted pools count matches the total primary site
+        # pools count.
         n_pools_demoted = len(result.results['output'].split('\n'))
-        self.assertEqual(len(site_a_pools), n_pools_demoted)
-        self.wait_for_mirror_state('up+unknown', model_name=self.site_a_model)
+        self.assertEqual(len(primary_site_pools), n_pools_demoted)
+
+        # At this point, both primary and secondary sites are demoted. Validate
+        # that the Ceph images, from both sites, report 'up+unknown', since
+        # there isn't a primary site at the moment.
+        logging.info('Waiting until {} is demoted.'.format(
+            primary_site_app_name))
         self.wait_for_mirror_state(
             'up+unknown',
-            application_name=self.application_name + self.site_b_app_suffix,
-            model_name=self.site_b_model)
+            application_name=primary_site_app_name,
+            model_name=primary_site_model,
+            pools=primary_site_pools)
+        self.wait_for_mirror_state(
+            'up+unknown',
+            application_name=secondary_site_app_name,
+            model_name=secondary_site_model,
+            pools=secondary_site_pools)
+
+        # Run the 'promote' Juju against the secondary site.
+        logging.info('Promoting {} from model {}.'.format(
+            secondary_site_app_name, secondary_site_model))
         result = zaza.model.run_action_on_leader(
-            'ceph-rbd-mirror' + self.site_b_app_suffix,
+            secondary_site_app_name,
             'promote',
-            model_name=self.site_b_model,
-            action_params={})
+            model_name=secondary_site_model,
+            action_params={
+                'pools': ','.join(secondary_site_pools)
+            })
         logging.info(result.results)
+        self.assertEqual(int(result.results['Code']), 0)
+
+        # Validate that the promoted pools count matches the total secondary
+        # site pools count.
         n_pools_promoted = len(result.results['output'].split('\n'))
-        self.assertEqual(len(site_b_pools), n_pools_promoted)
+        self.assertEqual(len(secondary_site_pools), n_pools_promoted)
+
+        # Validate that the Ceph images from the newly promoted site
+        # report 'up+stopped' state (which is reported by primary Ceph images).
+        logging.info('Waiting until {} is promoted.'.format(
+            secondary_site_app_name))
+        self.wait_for_mirror_state(
+            'up+stopped',
+            application_name=secondary_site_app_name,
+            model_name=secondary_site_model,
+            pools=secondary_site_pools)
+
+        # Validate that the Ceph images from site-a report 'up+replaying'
+        # (which is reported by secondary Ceph images).
         self.wait_for_mirror_state(
             'up+replaying',
-            model_name=self.site_a_model)
+            check_entries_behind_master=True,
+            application_name=primary_site_app_name,
+            model_name=primary_site_model,
+            pools=primary_site_pools)
+
+    def test_100_cinder_failover(self):
+        """Validate controlled failover via the Cinder API.
+
+        This test only makes sense if Cinder RBD mirroring mode is 'image'.
+        It will return early, if this is not the case.
+        """
+        cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode(
+            self.cinder_ceph_app_name)
+        if cinder_rbd_mirroring_mode != 'image':
+            logging.warning(
+                "Skipping 'test_100_cinder_failover' since Cinder RBD "
+                "mirroring mode is {}.".format(cinder_rbd_mirroring_mode))
+            return
+
+        session = openstack.get_overcloud_keystone_session()
+        cinder = openstack.get_cinder_session_client(session, version=3)
+
+        # Check if the Cinder volume host is available with replication
+        # enabled.
+        host = 'cinder@{}'.format(self.cinder_ceph_app_name)
+        svc = cinder.services.list(host=host, binary='cinder-volume')[0]
+        self.assertEqual(svc.replication_status, 'enabled')
+        self.assertEqual(svc.status, 'enabled')
+
+        # Setup the test Cinder volume
+        volume = self.setup_test_cinder_volume()
+
+        # Check if the volume is properly mirrored
         self.wait_for_mirror_state(
-            'up+stopped',
+            'up+replaying',
+            check_entries_behind_master=True,
             application_name=self.application_name + self.site_b_app_suffix,
-            model_name=self.site_b_model)
-        result = zaza.model.run_action_on_leader(
-            'ceph-rbd-mirror' + self.site_b_app_suffix,
-            'demote',
             model_name=self.site_b_model,
-            action_params={
-            })
-        logging.info(result.results)
-        n_pools_demoted = len(result.results['output'].split('\n'))
-        self.assertEqual(len(site_a_pools), n_pools_demoted)
-        self.wait_for_mirror_state(
-            'up+unknown',
-            model_name=self.site_a_model)
-        self.wait_for_mirror_state(
-            'up+unknown',
-            application_name=self.application_name + self.site_b_app_suffix,
-            model_name=self.site_b_model)
+            pools=[self.cinder_ceph_app_name])
+
+        # Execute the Cinder volume failover
+        openstack.failover_cinder_volume_host(
+            cinder=cinder,
+            backend_name=self.cinder_ceph_app_name,
+            target_backend_id='ceph',
+            target_status='disabled',
+            target_replication_status='failed-over')
+
+        # Check if the test volume is still available after failover
+        self.assertEqual(cinder.volumes.get(volume.id).status, 'available')
+
+    def test_101_cinder_failback(self):
+        """Validate controlled failback via the Cinder API.
+
+        This test only makes sense if Cinder RBD mirroring mode is 'image'.
+        It will return early, if this is not the case.
+
+        The test needs to be executed when the Cinder volume host is already
+        failed-over with the test volume on it.
+        """
+        cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode(
+            self.cinder_ceph_app_name)
+        if cinder_rbd_mirroring_mode != 'image':
+            logging.warning(
+                "Skipping 'test_101_cinder_failback' since Cinder RBD "
+                "mirroring mode is {}.".format(cinder_rbd_mirroring_mode))
+            return
+
+        session = openstack.get_overcloud_keystone_session()
+        cinder = openstack.get_cinder_session_client(session, version=3)
+
+        # Check if the Cinder volume host is already failed-over
+        host = 'cinder@{}'.format(self.cinder_ceph_app_name)
+        svc = cinder.services.list(host=host, binary='cinder-volume')[0]
+        self.assertEqual(svc.replication_status, 'failed-over')
+        self.assertEqual(svc.status, 'disabled')
+
+        # Check if the test Cinder volume is already present. The method
+        # 'cinder.volumes.find' raises 404 if the volume is not found.
+        volume = cinder.volumes.find(name=self.test_cinder_volume_name)
+
+        # Execute the Cinder volume failback
+        openstack.failover_cinder_volume_host(
+            cinder=cinder,
+            backend_name=self.cinder_ceph_app_name,
+            target_backend_id='default',
+            target_status='enabled',
+            target_replication_status='enabled')
+
+        # Check if the test volume is still available after failback
+        self.assertEqual(cinder.volumes.get(volume.id).status, 'available')
+
+    def test_200_juju_failover(self):
+        """Validate controlled failover via Juju actions."""
+        # Get the Ceph pools needed to failover
+        site_a_pools, site_b_pools = self.get_failover_pools()
+
+        # Execute the failover Juju actions with the appropriate parameters.
+        site_b_app_name = self.application_name + self.site_b_app_suffix
+        self.execute_failover_juju_actions(
+            primary_site_app_name=self.application_name,
+            primary_site_model=self.site_a_model,
+            primary_site_pools=site_a_pools,
+            secondary_site_app_name=site_b_app_name,
+            secondary_site_model=self.site_b_model,
+            secondary_site_pools=site_b_pools)
+
+    def test_201_juju_failback(self):
+        """Validate controlled failback via Juju actions."""
+        # Get the Ceph pools needed to failback
+        site_a_pools, site_b_pools = self.get_failover_pools()
+
+        # Execute the failover Juju actions with the appropriate parameters.
+        # The failback operation is just a failover with sites in reverse
+        # order.
+        site_b_app_name = self.application_name + self.site_b_app_suffix
+        self.execute_failover_juju_actions(
+            primary_site_app_name=site_b_app_name,
+            primary_site_model=self.site_b_model,
+            primary_site_pools=site_b_pools,
+            secondary_site_app_name=self.application_name,
+            secondary_site_model=self.site_a_model,
+            secondary_site_pools=site_a_pools)
+
+    def test_203_juju_resync(self):
+        """Validate the 'resync-pools' Juju action.
+
+        The 'resync-pools' Juju action is meant to flag Ceph images from the
+        secondary site to re-sync against the Ceph images from the primary
+        site.
+
+        This use case is useful when the Ceph secondary images are out of sync.
+        """
+        # Get the Ceph pools needed to failback
+        _, site_b_pools = self.get_failover_pools()
+
+        # Run the 'resync-pools' Juju action against the pools from site-b.
+        # This will make sure that the Ceph images from site-b are properly
+        # synced with the primary images from site-a.
+        site_b_app_name = self.application_name + self.site_b_app_suffix
+        logging.info('Re-syncing {} from model {}'.format(
+            site_b_app_name, self.site_b_model))
         result = zaza.model.run_action_on_leader(
-            'ceph-rbd-mirror',
-            'promote',
-            model_name=self.site_a_model,
-            action_params={
-            })
-        logging.info(result.results)
-        n_pools_promoted = len(result.results['output'].split('\n'))
-        self.assertEqual(len(site_b_pools), n_pools_promoted)
-        self.wait_for_mirror_state(
-            'up+stopped',
-            model_name=self.site_a_model)
-        result = zaza.model.run_action_on_leader(
-            'ceph-rbd-mirror' + self.site_b_app_suffix,
+            site_b_app_name,
             'resync-pools',
             model_name=self.site_b_model,
             action_params={
+                'pools': ','.join(site_b_pools),
                 'i-really-mean-it': True,
             })
         logging.info(result.results)
+        self.assertEqual(int(result.results['Code']), 0)
+
+        # Validate that the Ceph images from site-b report 'up+replaying'
+        # (which is reported by secondary Ceph images). And check that images
+        # exist in Cinder and Glance pools.
         self.wait_for_mirror_state(
             'up+replaying',
-            application_name=self.application_name + self.site_b_app_suffix,
+            check_entries_behind_master=True,
+            application_name=site_b_app_name,
             model_name=self.site_b_model,
-            require_images_in=['cinder-ceph', 'glance'])
+            require_images_in=[self.cinder_ceph_app_name, 'glance'],
+            pools=site_b_pools)
 
 
 class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase):
     """Encapsulate ``ceph-rbd-mirror`` destructive tests."""
 
-    def test_kill_site_a_fail_over(self):
-        """Validate fail over after uncontrolled shutdown of primary."""
-        for application in 'ceph-rbd-mirror', 'ceph-mon', 'ceph-osd':
+    def apply_cinder_ceph_workaround(self):
+        """Set minimal timeouts / retries to the Cinder Ceph backend.
+
+        This is needed because the failover via Cinder API will try to do a
+        demotion of the site-a. However, when site-a is down, and with the
+        default timeouts / retries, the operation takes an unreasonably amount
+        of time (or sometimes it never finishes).
+        """
+        # These new config options need to be set under the Cinder Ceph backend
+        # section in the main Cinder config file.
+        # At the moment, we don't the possibility of using Juju config to set
+        # these options. And also, it's not even a good practice to have them
+        # in production.
+        # These should be set only to do the Ceph failover via Cinder API, and
+        # they need to be removed after.
+        configs = {
+            'rados_connect_timeout': '1',
+            'rados_connection_retries': '1',
+            'rados_connection_interval': '0',
+            'replication_connect_timeout': '1',
+        }
+
+        # Small Python script that will be executed via Juju run to update
+        # the Cinder config file.
+        update_cinder_conf_script = (
+            "import configparser; "
+            "config = configparser.ConfigParser(); "
+            "config.read('/etc/cinder/cinder.conf'); "
+            "{}"
+            "f = open('/etc/cinder/cinder.conf', 'w'); "
+            "config.write(f); "
+            "f.close()")
+        set_cmd = ''
+        for cfg_name in configs:
+            set_cmd += "config.set('{0}', '{1}', '{2}'); ".format(
+                self.cinder_ceph_app_name, cfg_name, configs[cfg_name])
+        script = update_cinder_conf_script.format(set_cmd)
+
+        # Run the workaround script via Juju run
+        zaza.model.run_on_leader(
+            self.cinder_ceph_app_name,
+            'python3 -c "{}"; systemctl restart cinder-volume'.format(script))
+
+    def kill_primary_site(self):
+        """Simulate an unexpected primary site shutdown."""
+        logging.info('Killing the Ceph primary site')
+        for application in ['ceph-rbd-mirror', 'ceph-mon', 'ceph-osd']:
             zaza.model.remove_application(
                 application,
                 model_name=self.site_a_model,
                 forcefully_remove_machines=True)
+
+    def test_100_forced_juju_failover(self):
+        """Validate Ceph failover via Juju when the primary site is down.
+
+        * Kill the primary site
+        * Execute the forced failover via Juju actions
+        """
+        # Get the site-b Ceph pools that need to be promoted
+        _, site_b_pools = self.get_failover_pools()
+        site_b_app_name = self.application_name + self.site_b_app_suffix
+
+        # Simulate primary site unexpected shutdown
+        self.kill_primary_site()
+
+        # Try and promote the site-b to primary.
         result = zaza.model.run_action_on_leader(
-            'ceph-rbd-mirror' + self.site_b_app_suffix,
+            site_b_app_name,
             'promote',
             model_name=self.site_b_model,
             action_params={
+                'pools': ','.join(site_b_pools),
             })
+        self.assertEqual(int(result.results['Code']), 0)
+
+        # The site-b 'promote' Juju action is expected to fail, because the
+        # primary site is down.
         self.assertEqual(result.status, 'failed')
+
+        # Retry to promote site-b using the 'force' Juju action parameter.
         result = zaza.model.run_action_on_leader(
-            'ceph-rbd-mirror' + self.site_b_app_suffix,
+            site_b_app_name,
             'promote',
             model_name=self.site_b_model,
             action_params={
                 'force': True,
+                'pools': ','.join(site_b_pools),
             })
+        self.assertEqual(int(result.results['Code']), 0)
+
+        # Validate successful Juju action execution
         self.assertEqual(result.status, 'completed')
+
+    def test_200_forced_cinder_failover(self):
+        """Validate Ceph failover via Cinder when the primary site is down.
+
+        This test only makes sense if Cinder RBD mirroring mode is 'image'.
+        It will return early, if this is not the case.
+
+        This assumes that the primary site is already killed.
+        """
+        cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode(
+            self.cinder_ceph_app_name)
+        if cinder_rbd_mirroring_mode != 'image':
+            logging.warning(
+                "Skipping 'test_200_cinder_failover_without_primary_site' "
+                "since Cinder RBD mirroring mode is {}.".format(
+                    cinder_rbd_mirroring_mode))
+            return
+
+        # Make sure that the Cinder Ceph backend workaround is applied.
+        self.apply_cinder_ceph_workaround()
+
+        session = openstack.get_overcloud_keystone_session()
+        cinder = openstack.get_cinder_session_client(session, version=3)
+        openstack.failover_cinder_volume_host(
+            cinder=cinder,
+            backend_name=self.cinder_ceph_app_name,
+            target_backend_id='ceph',
+            target_status='disabled',
+            target_replication_status='failed-over')
+
+        # Check that the Cinder volumes are still available after forced
+        # failover.
+        for volume in cinder.volumes.list():
+            self.assertEqual(volume.status, 'available')

zaza/openstack/charm_tests/ceph/tests.py

@@ -33,7 +33,7 @@ import zaza.model as zaza_model
 import zaza.openstack.utilities.ceph as zaza_ceph
 import zaza.openstack.utilities.exceptions as zaza_exceptions
 import zaza.openstack.utilities.generic as zaza_utils
-import zaza.openstack.utilities.juju as zaza_juju
+import zaza.utilities.juju as juju_utils
 import zaza.openstack.utilities.openstack as zaza_openstack
 
 
@@ -124,7 +124,7 @@ class CephRelationTest(test_utils.OpenStackBaseTest):
         relation_name = 'osd'
         remote_unit = zaza_model.get_unit_from_name(remote_unit_name)
         remote_ip = remote_unit.public_address
-        relation = zaza_juju.get_relation_from_unit(
+        relation = juju_utils.get_relation_from_unit(
             unit_name,
             remote_unit_name,
             relation_name
@@ -153,7 +153,7 @@ class CephRelationTest(test_utils.OpenStackBaseTest):
             'ceph-public-address': remote_ip,
             'fsid': fsid,
         }
-        relation = zaza_juju.get_relation_from_unit(
+        relation = juju_utils.get_relation_from_unit(
             unit_name,
             remote_unit_name,
             relation_name
@@ -881,9 +881,13 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest):
     def setUpClass(cls):
         """Perform class one time initialization."""
         super(BlueStoreCompressionCharmOperation, cls).setUpClass()
+        release_application = 'keystone'
+        try:
+            zaza_model.get_application(release_application)
+        except KeyError:
+            release_application = 'ceph-mon'
         cls.current_release = zaza_openstack.get_os_release(
-            zaza_openstack.get_current_os_release_pair(
-                application='ceph-mon'))
+            application=release_application)
         cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky')
 
     def setUp(self):
@@ -976,7 +980,7 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest):
         ceph_pools_detail = zaza_ceph.get_ceph_pool_details(
             model_name=self.model_name)
         logging.debug('AFTER: {}'.format(ceph_pools_detail))
-        logging.debug(zaza_juju.get_relation_from_unit(
+        logging.debug(juju_utils.get_relation_from_unit(
             'ceph-mon', self.application_name, None,
             model_name=self.model_name))
         logging.info('Checking Ceph pool compression_mode after restoring '

zaza/openstack/charm_tests/charm_upgrade/tests.py

@@ -35,6 +35,7 @@ class FullCloudCharmUpgradeTest(unittest.TestCase):
         """Run setup for Charm Upgrades."""
         cli_utils.setup_logging()
         cls.lts = LTSGuestCreateTest()
+        cls.lts.setUpClass()
         cls.target_charm_namespace = '~openstack-charmers-next'
 
     def get_upgrade_url(self, charm_url):

zaza/openstack/charm_tests/cinder_backup/tests.py

@@ -63,7 +63,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
             self.cinder_client.volumes,
             vol_new.id,
             expected_status="available",
-            msg="Volume status wait")
+            msg="Extended volume")
 
     def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self):
         """Create, backup, delete, restore a ceph-backed cinder volume.
@@ -89,29 +89,44 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
 
         self.assertEqual(pool_name, expected_pool)
 
-        # Create ceph-backed cinder volume
-        cinder_vol = self.cinder_client.volumes.create(
-            name='{}-410-vol'.format(self.RESOURCE_PREFIX),
-            size=1)
-        openstack_utils.resource_reaches_status(
-            self.cinder_client.volumes,
-            cinder_vol.id,
-            wait_iteration_max_time=180,
-            stop_after_attempt=30,
-            expected_status='available',
-            msg='Volume status wait')
+        for attempt in tenacity.Retrying(
+                stop=tenacity.stop_after_attempt(3)):
+            with attempt:
+                # Create ceph-backed cinder volume
+                cinder_vol_name = '{}-410-{}-vol'.format(
+                    self.RESOURCE_PREFIX, attempt.retry_state.attempt_number)
+                cinder_vol = self.cinder_client.volumes.create(
+                    name=cinder_vol_name, size=1)
+                openstack_utils.resource_reaches_status(
+                    self.cinder_client.volumes,
+                    cinder_vol.id,
+                    wait_iteration_max_time=180,
+                    stop_after_attempt=15,
+                    expected_status='available',
+                    msg='ceph-backed cinder volume')
+
+                # Back up the volume
+                # NOTE(lourot): sometimes, especially on Mitaka, the backup
+                # remains stuck forever in 'creating' state and the volume in
+                # 'backing-up' state. See lp:1877076
+                # Attempting to create another volume and another backup
+                # usually then succeeds. Release notes and bug trackers show
+                # that many things have been fixed and are still left to be
+                # fixed in this area.
+                # When the backup creation succeeds, it usually does within
+                # 12 minutes.
+                vol_backup_name = '{}-410-{}-backup-vol'.format(
+                    self.RESOURCE_PREFIX, attempt.retry_state.attempt_number)
+                vol_backup = self.cinder_client.backups.create(
+                    cinder_vol.id, name=vol_backup_name)
+                openstack_utils.resource_reaches_status(
+                    self.cinder_client.backups,
+                    vol_backup.id,
+                    wait_iteration_max_time=180,
+                    stop_after_attempt=15,
+                    expected_status='available',
+                    msg='Backup volume')
 
-        # Backup the volume
-        vol_backup = self.cinder_client.backups.create(
-            cinder_vol.id,
-            name='{}-410-backup-vol'.format(self.RESOURCE_PREFIX))
-        openstack_utils.resource_reaches_status(
-            self.cinder_client.backups,
-            vol_backup.id,
-            wait_iteration_max_time=180,
-            stop_after_attempt=30,
-            expected_status='available',
-            msg='Volume status wait')
         # Delete the volume
         openstack_utils.delete_volume(self.cinder_client, cinder_vol.id)
         # Restore the volume
@@ -122,7 +137,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
             wait_iteration_max_time=180,
             stop_after_attempt=15,
             expected_status='available',
-            msg='Backup status wait')
+            msg='Restored backup volume')
         # Delete the backup
         openstack_utils.delete_volume_backup(
             self.cinder_client,
@@ -143,12 +158,12 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
         obj_count_samples.append(obj_count)
         pool_size_samples.append(kb_used)
 
-        name = '{}-410-vol'.format(self.RESOURCE_PREFIX)
         vols = self.cinder_client.volumes.list()
         try:
-            cinder_vols = [v for v in vols if v.name == name]
+            cinder_vols = [v for v in vols if v.name == cinder_vol_name]
         except AttributeError:
-            cinder_vols = [v for v in vols if v.display_name == name]
+            cinder_vols = [v for v in vols if
+                           v.display_name == cinder_vol_name]
         if not cinder_vols:
             # NOTE(hopem): it appears that at some point cinder-backup stopped
             # restoring volume metadata properly so revert to default name if

zaza/openstack/charm_tests/designate/tests.py

@@ -23,9 +23,9 @@ import designateclient.v1.records as records
 import designateclient.v1.servers as servers
 
 import zaza.model
-import zaza.openstack.utilities.juju as zaza_juju
-import zaza.openstack.utilities.generic as generic_utils
+import zaza.utilities.juju as juju_utils
 import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.openstack.utilities.generic as generic_utils
 import zaza.openstack.utilities.openstack as openstack_utils
 import zaza.openstack.charm_tests.designate.utils as designate_utils
 import zaza.charm_lifecycle.utils as lifecycle_utils
@@ -179,7 +179,7 @@ class DesignateAPITests(BaseDesignateTest):
         reraise=True
     )
     def _wait_to_resolve_test_record(self):
-        dns_ip = zaza_juju.get_relation_from_unit(
+        dns_ip = juju_utils.get_relation_from_unit(
             'designate/0',
             'designate-bind/0',
             'dns-backend'

zaza/openstack/charm_tests/glance/setup.py

@@ -14,7 +14,11 @@
 
 """Code for configuring glance."""
 
+import json
 import logging
+
+import boto3
+import zaza.model as model
 import zaza.openstack.utilities.openstack as openstack_utils
 import zaza.utilities.deployment_env as deployment_env
 
@@ -32,8 +36,37 @@ def basic_setup():
     """
 
 
+def _get_default_glance_client():
+    """Create default Glance client using overcloud credentials."""
+    keystone_session = openstack_utils.get_overcloud_keystone_session()
+    glance_client = openstack_utils.get_glance_session_client(keystone_session)
+    return glance_client
+
+
+def get_stores_info(glance_client=None):
+    """Retrieve glance backing store info.
+
+    :param glance_client: Authenticated glanceclient
+    :type glance_client: glanceclient.Client
+    """
+    glance_client = glance_client or _get_default_glance_client()
+    stores = glance_client.images.get_stores_info().get("stores", [])
+    return stores
+
+
+def get_store_ids(glance_client=None):
+    """Retrieve glance backing store ids.
+
+    :param glance_client: Authenticated glanceclient
+    :type glance_client: glanceclient.Client
+    """
+    stores = get_stores_info(glance_client)
+    return [store["id"] for store in stores]
+
+
 def add_image(image_url, glance_client=None, image_name=None, tags=[],
-              properties=None):
+              properties=None, backend=None, disk_format='qcow2',
+              visibility='public', container_format='bare'):
     """Retrieve image from ``image_url`` and add it to glance.
 
     :param image_url: Retrievable URL with image data
@@ -47,10 +80,14 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[],
     :param properties: Properties to add to image
     :type properties: dict
     """
-    if not glance_client:
-        keystone_session = openstack_utils.get_overcloud_keystone_session()
-        glance_client = openstack_utils.get_glance_session_client(
-            keystone_session)
+    glance_client = glance_client or _get_default_glance_client()
+    if backend is not None:
+        stores = get_store_ids(glance_client)
+        if backend not in stores:
+            raise ValueError("Invalid backend: %(backend)s "
+                             "(available: %(available)s)" % {
+                                 "backend": backend,
+                                 "available": ", ".join(stores)})
     if image_name:
         image = openstack_utils.get_images_by_name(
             glance_client, image_name)
@@ -65,7 +102,11 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[],
             image_url,
             image_name,
             tags=tags,
-            properties=properties)
+            properties=properties,
+            backend=backend,
+            disk_format=disk_format,
+            visibility=visibility,
+            container_format=container_format)
 
 
 def add_cirros_image(glance_client=None, image_name=None):
@@ -124,3 +165,58 @@ def add_lts_image(glance_client=None, image_name=None, release=None,
               glance_client=glance_client,
               image_name=image_name,
               properties=properties)
+
+
+def configure_external_s3_backend():
+    """Set up Ceph-radosgw as an external S3 backend for Glance."""
+    logging.info("Creating a test S3 user and credentials for Glance")
+    username, displayname = "zaza-glance-test", "Zaza Glance Test User"
+    cmd = "radosgw-admin user create --uid='{}' --display-name='{}'".format(
+        username, displayname
+    )
+    results = model.run_on_leader("ceph-mon", cmd)
+    stdout = json.loads(results["stdout"])
+    keys = stdout["keys"][0]
+    access_key, secret_key = keys["access_key"], keys["secret_key"]
+
+    logging.info("Getting S3 endpoint URL of Radosgw from Keystone")
+    keystone_auth = openstack_utils.get_overcloud_auth()
+    keystone_client = openstack_utils.get_keystone_client(keystone_auth)
+    endpoint_url = keystone_client.session.get_endpoint(
+        service_type="s3",
+        interface="public",
+        region="RegionOne",
+    )
+
+    logging.info("Creating a test S3 bucket for Glance")
+    bucket_name = "zaza-glance-s3-test"
+    s3_client = boto3.client(
+        "s3",
+        endpoint_url=endpoint_url,
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+    )
+    s3_client.create_bucket(Bucket=bucket_name)
+
+    logging.info("Updating Glance configs with S3 endpoint information")
+    model.set_application_config(
+        "glance",
+        {
+            "s3-store-host": endpoint_url,
+            "s3-store-access-key": access_key,
+            "s3-store-secret-key": secret_key,
+            "s3-store-bucket": bucket_name,
+        },
+    )
+    model.wait_for_agent_status()
+
+    logging.info("Waiting for units to reach target states")
+    model.wait_for_application_states(
+        states={
+            "glance": {
+                "workload-status": "active",
+                "workload-status-message": "Unit is ready",
+            }
+        }
+    )
+    model.block_until_all_units_idle()

zaza/openstack/charm_tests/glance/tests.py

@@ -18,8 +18,10 @@
 
 import logging
 
-import zaza.openstack.utilities.openstack as openstack_utils
+import boto3
+import zaza.model as model
 import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.openstack.utilities.openstack as openstack_utils
 
 
 class GlanceTest(test_utils.OpenStackBaseTest):
@@ -67,6 +69,32 @@ class GlanceTest(test_utils.OpenStackBaseTest):
             {'image_format': {'disk_formats': ['qcow2']}},
             ['glance-api'])
 
+    def test_412_image_conversion(self):
+        """Check image-conversion config.
+
+        When image-conversion config is enabled glance will convert images
+        to raw format, this is only performed for interoperable image import
+        docs.openstack.org/glance/train/admin/interoperable-image-import.html
+        image conversion is done at server-side for better image handling
+        """
+        current_release = openstack_utils.get_os_release()
+        bionic_stein = openstack_utils.get_os_release('bionic_stein')
+        if current_release < bionic_stein:
+            self.skipTest('image-conversion config is supported since '
+                          'bionic_stein or newer versions')
+
+        with self.config_change({'image-conversion': 'false'},
+                                {'image-conversion': 'true'}):
+            image_url = openstack_utils.find_cirros_image(arch='x86_64')
+            image = openstack_utils.create_image(
+                self.glance_client,
+                image_url,
+                'cirros-test-import',
+                force_import=True)
+
+            disk_format = self.glance_client.images.get(image.id).disk_format
+            self.assertEqual('raw', disk_format)
+
     def test_900_restart_on_config_change(self):
         """Checking restart happens on config change."""
         # Config file affected by juju set config change
@@ -92,3 +120,105 @@ class GlanceTest(test_utils.OpenStackBaseTest):
         they are started
         """
         self.pause_resume(['glance-api'])
+
+
+class GlanceCephRGWBackendTest(test_utils.OpenStackBaseTest):
+    """Encapsulate glance tests using the Ceph RGW backend.
+
+    It validates the Ceph RGW backend in glance, which uses the Swift API.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running glance tests."""
+        super(GlanceCephRGWBackendTest, cls).setUpClass()
+
+        swift_session = openstack_utils.get_keystone_session_from_relation(
+            'ceph-radosgw')
+        cls.swift = openstack_utils.get_swift_session_client(
+            swift_session)
+        cls.glance_client = openstack_utils.get_glance_session_client(
+            cls.keystone_session)
+
+    def test_100_create_image(self):
+        """Create an image and do a simple validation of it.
+
+        The OpenStack Swift API is used to do the validation, since the Ceph
+        Rados Gateway serves an API which is compatible with that.
+        """
+        image_name = 'zaza-ceph-rgw-image'
+        openstack_utils.create_image(
+            glance=self.glance_client,
+            image_url=openstack_utils.find_cirros_image(arch='x86_64'),
+            image_name=image_name,
+            backend='swift')
+        headers, containers = self.swift.get_account()
+        self.assertEqual(len(containers), 1)
+        container_name = containers[0].get('name')
+        headers, objects = self.swift.get_container(container_name)
+        images = openstack_utils.get_images_by_name(
+            self.glance_client,
+            image_name)
+        self.assertEqual(len(images), 1)
+        image = images[0]
+        total_bytes = 0
+        for ob in objects:
+            if '{}-'.format(image['id']) in ob['name']:
+                total_bytes = total_bytes + int(ob['bytes'])
+        logging.info(
+            'Checking glance image size {} matches swift '
+            'image size {}'.format(image['size'], total_bytes))
+        self.assertEqual(image['size'], total_bytes)
+        openstack_utils.delete_image(self.glance_client, image['id'])
+
+
+class GlanceExternalS3Test(test_utils.OpenStackBaseTest):
+    """Encapsulate glance tests using an external S3 backend."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running glance tests with S3 backend."""
+        super(GlanceExternalS3Test, cls).setUpClass()
+        cls.glance_client = openstack_utils.get_glance_session_client(
+            cls.keystone_session
+        )
+
+        configs = model.get_application_config("glance")
+        cls.s3_store_host = configs["s3-store-host"]["value"]
+        cls.s3_store_access_key = configs["s3-store-access-key"]["value"]
+        cls.s3_store_secret_key = configs["s3-store-secret-key"]["value"]
+        cls.s3_store_bucket = configs["s3-store-bucket"]["value"]
+
+    def test_100_create_delete_image(self):
+        """Create an image and do a simple validation of it.
+
+        Validate the size of the image in both Glance API and actual S3 bucket.
+        """
+        image_name = "zaza-s3-test-image"
+        openstack_utils.create_image(
+            glance=self.glance_client,
+            image_url=openstack_utils.find_cirros_image(arch="x86_64"),
+            image_name=image_name,
+            backend="s3",
+        )
+        images = openstack_utils.get_images_by_name(
+            self.glance_client, image_name
+        )
+        self.assertEqual(len(images), 1)
+        image = images[0]
+
+        s3_client = boto3.client(
+            "s3",
+            endpoint_url=self.s3_store_host,
+            aws_access_key_id=self.s3_store_access_key,
+            aws_secret_access_key=self.s3_store_secret_key,
+        )
+        response = s3_client.head_object(
+            Bucket=self.s3_store_bucket, Key=image["id"]
+        )
+        logging.info(
+            "Checking glance image size {} matches S3 object's ContentLength "
+            "{}".format(image["size"], response["ContentLength"])
+        )
+        self.assertEqual(image["size"], response["ContentLength"])
+        openstack_utils.delete_image(self.glance_client, image["id"])

zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py

@@ -17,9 +17,32 @@
 """Code for configuring glance-simplestreams-sync."""
 
 import logging
+import tenacity
+import pprint
 
 import zaza.model as zaza_model
 import zaza.openstack.utilities.generic as generic_utils
+import zaza.openstack.utilities.openstack as openstack_utils
+
+
+def _get_catalog():
+    """Retrieve the Keystone service catalog.
+
+    :returns: The raw Keystone service catalog.
+    :rtype: List[Dict]
+    """
+    keystone_session = openstack_utils.get_overcloud_keystone_session()
+    keystone_client = openstack_utils.get_keystone_session_client(
+        keystone_session)
+
+    token = keystone_session.get_token()
+    token_data = keystone_client.tokens.get_token_data(token)
+
+    if 'catalog' not in token_data['token']:
+        raise ValueError('catalog not in token data: "{}"'
+                         .format(pprint.pformat(token_data)))
+
+    return token_data['token']['catalog']
 
 
 def sync_images():
@@ -30,11 +53,27 @@ def sync_images():
     deployment.
     """
     logging.info("Synchronising images using glance-simplestreams-sync")
-    generic_utils.assertActionRanOK(
-        zaza_model.run_action_on_leader(
-            "glance-simplestreams-sync",
-            "sync-images",
-            raise_on_failure=True,
-            action_params={},
-        )
-    )
+
+    catalog = None
+    try:
+        for attempt in tenacity.Retrying(
+                stop=tenacity.stop_after_attempt(3),
+                wait=tenacity.wait_exponential(
+                    multiplier=1, min=2, max=10),
+                reraise=True):
+            with attempt:
+                # Proactively retrieve the Keystone service catalog so that we
+                # can log it in the event of a failure.
+                catalog = _get_catalog()
+                generic_utils.assertActionRanOK(
+                    zaza_model.run_action_on_leader(
+                        "glance-simplestreams-sync",
+                        "sync-images",
+                        raise_on_failure=True,
+                        action_params={},
+                    )
+                )
+    except Exception:
+        logging.info('Contents of Keystone service catalog: "{}"'
+                     .format(pprint.pformat(catalog)))
+        raise

zaza/openstack/charm_tests/hacluster/tests.py

@@ -76,57 +76,121 @@ class HaclusterTest(HaclusterBaseTest):
         self._toggle_maintenance_and_wait('false')
 
 
-class HaclusterScalebackTest(HaclusterBaseTest):
-    """hacluster scaleback tests."""
+class HaclusterScaleBackAndForthTest(HaclusterBaseTest):
+    """hacluster tests scaling back and forth."""
 
     @classmethod
     def setUpClass(cls):
-        """Run class setup for running hacluster scaleback tests."""
-        super(HaclusterScalebackTest, cls).setUpClass()
+        """Run class setup for running hacluster tests."""
+        super(HaclusterScaleBackAndForthTest, cls).setUpClass()
         test_config = cls.test_config['tests_options']['hacluster']
         cls._principle_app_name = test_config['principle-app-name']
         cls._hacluster_charm_name = test_config['hacluster-charm-name']
 
     def test_930_scaleback(self):
-        """Remove a unit and add a new one."""
+        """Remove one unit, recalculate quorum and re-add one unit.
+
+        NOTE(lourot): before lp:1400481 was fixed, the corosync ring wasn't
+        recalculated when removing units. So within a cluster of 3 units,
+        removing a unit and re-adding one led to a situation where corosync
+        considers having 3 nodes online out of 4, instead of just 3 out of 3.
+        This test covers this scenario.
+        """
         principle_units = sorted(zaza.model.get_status().applications[
             self._principle_app_name]['units'].keys())
         self.assertEqual(len(principle_units), 3)
-        doomed_principle_unit = principle_units[0]
-        other_principle_unit = principle_units[1]
+        surviving_principle_unit = principle_units[0]
+        doomed_principle_unit = principle_units[1]
+        surviving_hacluster_unit = juju_utils.get_subordinate_units(
+            [surviving_principle_unit],
+            charm_name=self._hacluster_charm_name)[0]
         doomed_hacluster_unit = juju_utils.get_subordinate_units(
-            [doomed_principle_unit], charm_name=self._hacluster_charm_name)[0]
-        other_hacluster_unit = juju_utils.get_subordinate_units(
-            [other_principle_unit], charm_name=self._hacluster_charm_name)[0]
+            [doomed_principle_unit],
+            charm_name=self._hacluster_charm_name)[0]
 
         logging.info('Pausing unit {}'.format(doomed_hacluster_unit))
         zaza.model.run_action(
             doomed_hacluster_unit,
             'pause',
             raise_on_failure=True)
-        logging.info('OK')
 
         logging.info('Removing {}'.format(doomed_principle_unit))
         zaza.model.destroy_unit(
             self._principle_app_name,
             doomed_principle_unit,
             wait_disappear=True)
-        logging.info('OK')
 
         logging.info('Waiting for model to settle')
-        zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked')
-        zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked')
+        zaza.model.block_until_unit_wl_status(surviving_hacluster_unit,
+                                              'blocked')
+        # NOTE(lourot): the surviving principle units (usually keystone units)
+        # aren't guaranteed to be blocked, so we don't validate that here.
         zaza.model.block_until_all_units_idle()
-        logging.info('OK')
 
-        logging.info('Adding an hacluster unit')
+        # At this point the corosync ring hasn't been updated yet, so it should
+        # still remember the deleted unit:
+        self.__assert_some_corosync_nodes_are_offline(surviving_hacluster_unit)
+
+        logging.info('Updating corosync ring')
+        hacluster_app_name = zaza.model.get_unit_from_name(
+            surviving_hacluster_unit).application
+        zaza.model.run_action_on_leader(
+            hacluster_app_name,
+            'update-ring',
+            action_params={'i-really-mean-it': True},
+            raise_on_failure=True)
+
+        # At this point if the corosync ring has been properly updated, there
+        # shouldn't be any trace of the deleted unit anymore:
+        self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit)
+
+        logging.info('Re-adding an hacluster unit')
         zaza.model.add_unit(self._principle_app_name, wait_appear=True)
-        logging.info('OK')
 
         logging.info('Waiting for model to settle')
-        zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active')
-        # NOTE(lourot): the principle application remains blocked after scaling
-        # back up until lp:1400481 is solved.
-        zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked')
+        # NOTE(lourot): the principle charm may remain blocked here. This seems
+        # to happen often when it is keystone and has a mysql-router as other
+        # subordinate charm. The keystone units seems to often remain blocked
+        # with 'Database not initialised'. This is not the hacluster charm's
+        # fault and this is why we don't validate here that the entire model
+        # goes back to active/idle.
+        zaza.model.block_until_unit_wl_status(surviving_hacluster_unit,
+                                              'active')
         zaza.model.block_until_all_units_idle()
-        logging.debug('OK')
+
+        # Because of lp:1874719 the corosync ring may show a mysterious offline
+        # 'node1' node. We clean up the ring by re-running the 'update-ring'
+        # action:
+        logging.info('Updating corosync ring - workaround for lp:1874719')
+        zaza.model.run_action_on_leader(
+            hacluster_app_name,
+            'update-ring',
+            action_params={'i-really-mean-it': True},
+            raise_on_failure=True)
+
+        # At this point the corosync ring should not contain any offline node:
+        self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit)
+
+    def __assert_some_corosync_nodes_are_offline(self, hacluster_unit):
+        logging.info('Checking that corosync considers at least one node to '
+                     'be offline')
+        output = self._get_crm_status(hacluster_unit)
+        self.assertIn('OFFLINE', output,
+                      "corosync should list at least one offline node")
+
+    def __assert_all_corosync_nodes_are_online(self, hacluster_unit):
+        logging.info('Checking that corosync considers all nodes to be online')
+        output = self._get_crm_status(hacluster_unit)
+        self.assertNotIn('OFFLINE', output,
+                         "corosync shouldn't list any offline node")
+
+    @staticmethod
+    def _get_crm_status(hacluster_unit):
+        cmd = 'sudo crm status'
+        result = zaza.model.run_on_unit(hacluster_unit, cmd)
+        code = result.get('Code')
+        if code != '0':
+            raise zaza.model.CommandRunFailed(cmd, result)
+        output = result.get('Stdout').strip()
+        logging.debug('crm output received: {}'.format(output))
+        return output

zaza/openstack/charm_tests/ironic/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Collection of code for setting up and testing ironic."""

zaza/openstack/charm_tests/ironic/setup.py

@@ -0,0 +1,184 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Code for configuring ironic."""
+
+import copy
+import os
+import tenacity
+
+import zaza.openstack.charm_tests.glance.setup as glance_setup
+import zaza.openstack.utilities.openstack as openstack_utils
+from zaza.openstack.utilities import (
+    cli as cli_utils,
+)
+import zaza.model as zaza_model
+
+
+FLAVORS = {
+    'bm1.small': {
+        'flavorid': 2,
+        'ram': 2048,
+        'disk': 20,
+        'vcpus': 1,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL1_SMALL": 1,
+        },
+    },
+    'bm1.medium': {
+        'flavorid': 3,
+        'ram': 4096,
+        'disk': 40,
+        'vcpus': 2,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL1_MEDIUM": 1,
+        },
+    },
+    'bm1.large': {
+        'flavorid': 4,
+        'ram': 8192,
+        'disk': 40,
+        'vcpus': 4,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL1_LARGE": 1,
+        },
+    },
+    'bm1.tempest': {
+        'flavorid': 6,
+        'ram': 256,
+        'disk': 1,
+        'vcpus': 1,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL1_TEMPEST": 1,
+        },
+    },
+    'bm2.tempest': {
+        'flavorid': 7,
+        'ram': 512,
+        'disk': 1,
+        'vcpus': 1,
+        'properties': {
+            "resources:CUSTOM_BAREMETAL2_TEMPEST": 1,
+        },
+    },
+}
+
+
+def _add_image(url, image_name, backend="swift",
+               disk_format="raw", container_format="bare"):
+    for attempt in tenacity.Retrying(
+            stop=tenacity.stop_after_attempt(3),
+            reraise=True):
+        with attempt:
+            glance_setup.add_image(
+                url,
+                image_name=image_name,
+                backend=backend,
+                disk_format=disk_format,
+                container_format=container_format)
+
+
+def add_ironic_deployment_image(initrd_url=None, kernel_url=None):
+    """Add Ironic deploy images to glance.
+
+    :param initrd_url: URL where the ari image resides
+    :type initrd_url: str
+    :param kernel_url: URL where the aki image resides
+    :type kernel_url: str
+    """
+    base_name = 'ironic-deploy'
+    initrd_name = "{}-initrd".format(base_name)
+    vmlinuz_name = "{}-vmlinuz".format(base_name)
+    if not initrd_url:
+        initrd_url = os.environ.get('TEST_IRONIC_DEPLOY_INITRD', None)
+    if not kernel_url:
+        kernel_url = os.environ.get('TEST_IRONIC_DEPLOY_VMLINUZ', None)
+    if not all([initrd_url, kernel_url]):
+        raise ValueError("Missing required deployment image URLs")
+
+    _add_image(
+        initrd_url,
+        initrd_name,
+        backend="swift",
+        disk_format="ari",
+        container_format="ari")
+
+    _add_image(
+        kernel_url,
+        vmlinuz_name,
+        backend="swift",
+        disk_format="aki",
+        container_format="aki")
+
+
+def add_ironic_os_image(image_url=None):
+    """Upload the operating system images built for bare metal deployments.
+
+    :param image_url: URL where the image resides
+    :type image_url: str
+    """
+    image_url = image_url or os.environ.get(
+        'TEST_IRONIC_RAW_BM_IMAGE', None)
+    image_name = "baremetal-ubuntu-image"
+    if image_url is None:
+        raise ValueError("Missing image_url")
+
+    _add_image(
+        image_url,
+        image_name,
+        backend="swift",
+        disk_format="raw",
+        container_format="bare")
+
+
+def set_temp_url_secret():
+    """Run the set-temp-url-secret on the ironic-conductor leader.
+
+    This is needed if direct boot method is enabled.
+    """
+    zaza_model.run_action_on_leader(
+        'ironic-conductor',
+        'set-temp-url-secret',
+        action_params={})
+
+
+def create_bm_flavors(nova_client=None):
+    """Create baremetal flavors.
+
+    :param nova_client: Authenticated nova client
+    :type nova_client: novaclient.v2.client.Client
+    """
+    if not nova_client:
+        keystone_session = openstack_utils.get_overcloud_keystone_session()
+        nova_client = openstack_utils.get_nova_session_client(
+            keystone_session)
+    cli_utils.setup_logging()
+    names = [flavor.name for flavor in nova_client.flavors.list()]
+    # Disable scheduling based on standard flavor properties
+    default_properties = {
+        "resources:VCPU": 0,
+        "resources:MEMORY_MB": 0,
+        "resources:DISK_GB": 0,
+    }
+    for flavor in FLAVORS.keys():
+        if flavor not in names:
+            properties = copy.deepcopy(default_properties)
+            properties.update(FLAVORS[flavor]["properties"])
+            bm_flavor = nova_client.flavors.create(
+                name=flavor,
+                ram=FLAVORS[flavor]['ram'],
+                vcpus=FLAVORS[flavor]['vcpus'],
+                disk=FLAVORS[flavor]['disk'],
+                flavorid=FLAVORS[flavor]['flavorid'])
+            bm_flavor.set_keys(properties)

zaza/openstack/charm_tests/ironic/tests.py

@@ -0,0 +1,83 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Encapsulate ironic testing."""
+
+import logging
+
+import ironicclient.client as ironic_client
+import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.openstack.utilities.openstack as openstack_utils
+
+
+def _get_ironic_client(ironic_api_version="1.58"):
+    keystone_session = openstack_utils.get_overcloud_keystone_session()
+    ironic = ironic_client.Client(1, session=keystone_session,
+                                  os_ironic_api_version=ironic_api_version)
+    return ironic
+
+
+class IronicTest(test_utils.OpenStackBaseTest):
+    """Run Ironic specific tests."""
+
+    _SERVICES = ['ironic-api']
+
+    def test_110_catalog_endpoints(self):
+        """Verify that the endpoints are present in the catalog."""
+        overcloud_auth = openstack_utils.get_overcloud_auth()
+        keystone_client = openstack_utils.get_keystone_client(
+            overcloud_auth)
+        actual_endpoints = keystone_client.service_catalog.get_endpoints()
+        actual_interfaces = [endpoint['interface'] for endpoint in
+                             actual_endpoints["baremetal"]]
+        for expected_interface in ('internal', 'admin', 'public'):
+            assert(expected_interface in actual_interfaces)
+
+    def test_400_api_connection(self):
+        """Simple api calls to check service is up and responding."""
+        ironic = _get_ironic_client()
+
+        logging.info('listing conductors')
+        conductors = ironic.conductor.list()
+        assert(len(conductors) > 0)
+
+        # By default, only IPMI HW type is enabled. iDrac and Redfish
+        # can optionally be enabled
+        drivers = ironic.driver.list()
+        driver_names = [drv.name for drv in drivers]
+
+        expected = ['intel-ipmi', 'ipmi']
+        for exp in expected:
+            assert(exp in driver_names)
+        assert(len(driver_names) == 2)
+
+    def test_900_restart_on_config_change(self):
+        """Checking restart happens on config change.
+
+        Change debug mode and assert that change propagates to the correct
+        file and that services are restarted as a result
+        """
+        self.restart_on_changed_debug_oslo_config_file(
+            '/etc/ironic/ironic.conf', self._SERVICES)
+
+    def test_910_pause_resume(self):
+        """Run pause and resume tests.
+
+        Pause service and check services are stopped then resume and check
+        they are started
+        """
+        logging.info('Skipping pause resume test LP: #1886202...')
+        return
+        with self.pause_resume(self._SERVICES):
+            logging.info("Testing pause resume")

zaza/openstack/charm_tests/kerberos/setup.py

@@ -113,7 +113,15 @@ def retrieve_and_attach_keytab():
                                    'keystone_keytab',
                                    tmp_file)
 
-    zaza.model.wait_for_application_states()
+    # cs:ubuntu charm has changed behaviour and we can't rely on the workload
+    # staus message.  Thus, ignore it.
+    states = {
+        "ubuntu-test-host": {
+            "workload-status": "active",
+            "workload-status-message": "",
+        }
+    }
+    zaza.model.wait_for_application_states(states=states)
     zaza.model.block_until_all_units_idle()
 
 

zaza/openstack/charm_tests/keystone/setup.py

@@ -41,9 +41,8 @@ def wait_for_cacert(model_name=None):
     :type model_name: str
     """
     logging.info("Waiting for cacert")
-    zaza.model.block_until_file_has_contents(
+    zaza.openstack.utilities.openstack.block_until_ca_exists(
         'keystone',
-        openstack_utils.KEYSTONE_REMOTE_CACERT,
         'CERTIFICATE',
         model_name=model_name)
     zaza.model.block_until_all_units_idle(model_name=model_name)

zaza/openstack/charm_tests/keystone/tests.py

@@ -229,7 +229,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest):
                     'OS_DOMAIN_NAME': DEMO_DOMAIN,
                 }
                 if self.tls_rid:
-                    openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT
+                    openrc['OS_CACERT'] = openstack_utils.get_cacert()
                     openrc['OS_AUTH_URL'] = (
                         openrc['OS_AUTH_URL'].replace('http', 'https'))
                 logging.info('keystone IP {}'.format(ip))
@@ -259,7 +259,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest):
         """
         def _validate_token_data(openrc):
             if self.tls_rid:
-                openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT
+                openrc['OS_CACERT'] = openstack_utils.get_cacert()
                 openrc['OS_AUTH_URL'] = (
                     openrc['OS_AUTH_URL'].replace('http', 'https'))
             logging.info('keystone IP {}'.format(ip))
@@ -380,7 +380,12 @@ class SecurityTests(BaseKeystoneTest):
 
 
 class LdapTests(BaseKeystoneTest):
-    """Keystone ldap tests tests."""
+    """Keystone ldap tests."""
+
+    non_string_type_keys = ('ldap-user-enabled-mask',
+                            'ldap-user-enabled-invert',
+                            'ldap-group-members-are-ids',
+                            'ldap-use-pool')
 
     @classmethod
     def setUpClass(cls):
@@ -402,13 +407,22 @@ class LdapTests(BaseKeystoneTest):
             'ldap-password': 'crapper',
             'ldap-suffix': 'dc=test,dc=com',
             'domain-name': 'userdomain',
+            'ldap-config-flags':
+                {
+                    'group_tree_dn': 'ou=groups,dc=test,dc=com',
+                    'group_objectclass': 'posixGroup',
+                    'group_name_attribute': 'cn',
+                    'group_member_attribute': 'memberUid',
+                    'group_members_are_ids': 'true',
+            }
         }
 
-    def _find_keystone_v3_user(self, username, domain):
+    def _find_keystone_v3_user(self, username, domain, group=None):
         """Find a user within a specified keystone v3 domain.
 
         :param str username: Username to search for in keystone
         :param str domain: username selected from which domain
+        :param str group: group to search for in keystone for group membership
         :return: return username if found
         :rtype: Optional[str]
         """
@@ -418,9 +432,15 @@ class LdapTests(BaseKeystoneTest):
                 openstack_utils.get_overcloud_auth(address=ip))
             client = openstack_utils.get_keystone_session_client(session)
 
-            domain_users = client.users.list(
-                domain=client.domains.find(name=domain).id
-            )
+            if group is None:
+                domain_users = client.users.list(
+                    domain=client.domains.find(name=domain).id,
+                )
+            else:
+                domain_users = client.users.list(
+                    domain=client.domains.find(name=domain).id,
+                    group=self._find_keystone_v3_group(group, domain).id,
+                )
 
             usernames = [u.name.lower() for u in domain_users]
             if username.lower() in usernames:
@@ -431,27 +451,232 @@ class LdapTests(BaseKeystoneTest):
         )
         return None
 
+    def _find_keystone_v3_group(self, group, domain):
+        """Find a group within a specified keystone v3 domain.
+
+        :param str group: Group to search for in keystone
+        :param str domain: group selected from which domain
+        :return: return group if found
+        :rtype: Optional[str]
+        """
+        for ip in self.keystone_ips:
+            logging.info('Keystone IP {}'.format(ip))
+            session = openstack_utils.get_keystone_session(
+                openstack_utils.get_overcloud_auth(address=ip))
+            client = openstack_utils.get_keystone_session_client(session)
+
+            domain_groups = client.groups.list(
+                domain=client.domains.find(name=domain).id
+            )
+
+            for searched_group in domain_groups:
+                if searched_group.name.lower() == group.lower():
+                    return searched_group
+
+        logging.debug(
+            "Group {} was not found. Returning None.".format(group)
+        )
+        return None
+
     def test_100_keystone_ldap_users(self):
         """Validate basic functionality of keystone API with ldap."""
         application_name = 'keystone-ldap'
-        config = self._get_ldap_config()
+        intended_cfg = self._get_ldap_config()
+        current_cfg, non_string_cfg = (
+            self.config_current_separate_non_string_type_keys(
+                self.non_string_type_keys, intended_cfg, application_name)
+        )
 
         with self.config_change(
-                self.config_current(application_name, config.keys()),
-                config,
-                application_name=application_name):
-            logging.info(
-                'Waiting for users to become available in keystone...'
-            )
-            test_config = lifecycle_utils.get_charm_config(fatal=False)
-            zaza.model.wait_for_application_states(
-                states=test_config.get("target_deploy_status", {})
-            )
+                {},
+                non_string_cfg,
+                application_name=application_name,
+                reset_to_charm_default=True):
+            with self.config_change(
+                    current_cfg,
+                    intended_cfg,
+                    application_name=application_name):
+                logging.info(
+                    'Waiting for users to become available in keystone...'
+                )
+                test_config = lifecycle_utils.get_charm_config(fatal=False)
+                zaza.model.wait_for_application_states(
+                    states=test_config.get("target_deploy_status", {})
+                )
 
-            with self.v3_keystone_preferred():
-                # NOTE(jamespage): Test fixture should have johndoe and janedoe
-                #                  accounts
-                johndoe = self._find_keystone_v3_user('john doe', 'userdomain')
-                self.assertIsNotNone(johndoe, "user 'john doe' was unknown")
-                janedoe = self._find_keystone_v3_user('jane doe', 'userdomain')
-                self.assertIsNotNone(janedoe, "user 'jane doe' was unknown")
+                with self.v3_keystone_preferred():
+                    # NOTE(jamespage): Test fixture should have
+                    #                  johndoe and janedoe accounts
+                    johndoe = self._find_keystone_v3_user(
+                        'john doe', 'userdomain')
+                    self.assertIsNotNone(
+                        johndoe, "user 'john doe' was unknown")
+                    janedoe = self._find_keystone_v3_user(
+                        'jane doe', 'userdomain')
+                    self.assertIsNotNone(
+                        janedoe, "user 'jane doe' was unknown")
+
+    def test_101_keystone_ldap_groups(self):
+        """Validate basic functionality of keystone API with ldap."""
+        application_name = 'keystone-ldap'
+        intended_cfg = self._get_ldap_config()
+        current_cfg, non_string_cfg = (
+            self.config_current_separate_non_string_type_keys(
+                self.non_string_type_keys, intended_cfg, application_name)
+        )
+
+        with self.config_change(
+                {},
+                non_string_cfg,
+                application_name=application_name,
+                reset_to_charm_default=True):
+            with self.config_change(
+                    current_cfg,
+                    intended_cfg,
+                    application_name=application_name):
+                logging.info(
+                    'Waiting for groups to become available in keystone...'
+                )
+                test_config = lifecycle_utils.get_charm_config(fatal=False)
+                zaza.model.wait_for_application_states(
+                    states=test_config.get("target_deploy_status", {})
+                )
+
+                with self.v3_keystone_preferred():
+                    # NOTE(arif-ali): Test fixture should have openstack and
+                    #                 admin groups
+                    openstack_group = self._find_keystone_v3_group(
+                        'openstack', 'userdomain')
+                    self.assertIsNotNone(
+                        openstack_group.name, "group 'openstack' was unknown")
+                    admin_group = self._find_keystone_v3_group(
+                        'admin', 'userdomain')
+                    self.assertIsNotNone(
+                        admin_group.name, "group 'admin' was unknown")
+
+    def test_102_keystone_ldap_group_membership(self):
+        """Validate basic functionality of keystone API with ldap."""
+        application_name = 'keystone-ldap'
+        intended_cfg = self._get_ldap_config()
+        current_cfg, non_string_cfg = (
+            self.config_current_separate_non_string_type_keys(
+                self.non_string_type_keys, intended_cfg, application_name)
+        )
+
+        with self.config_change(
+                {},
+                non_string_cfg,
+                application_name=application_name,
+                reset_to_charm_default=True):
+            with self.config_change(
+                    current_cfg,
+                    intended_cfg,
+                    application_name=application_name):
+                logging.info(
+                    'Waiting for groups to become available in keystone...'
+                )
+                test_config = lifecycle_utils.get_charm_config(fatal=False)
+                zaza.model.wait_for_application_states(
+                    states=test_config.get("target_deploy_status", {})
+                )
+
+                with self.v3_keystone_preferred():
+                    # NOTE(arif-ali): Test fixture should have openstack and
+                    #                 admin groups
+                    openstack_group = self._find_keystone_v3_user(
+                        'john doe', 'userdomain', group='openstack')
+                    self.assertIsNotNone(
+                        openstack_group,
+                        "john doe was not in group 'openstack'")
+                    admin_group = self._find_keystone_v3_user(
+                        'john doe', 'userdomain', group='admin')
+                    self.assertIsNotNone(
+                        admin_group, "'john doe' was not in group 'admin'")
+
+
+class LdapExplicitCharmConfigTests(LdapTests):
+    """Keystone ldap tests."""
+
+    def _get_ldap_config(self):
+        """Generate ldap config for current model.
+
+        :return: tuple of whether ldap-server is running and if so, config
+            for the keystone-ldap application.
+        :rtype: Tuple[bool, Dict[str,str]]
+        """
+        ldap_ips = zaza.model.get_app_ips("ldap-server")
+        self.assertTrue(ldap_ips, "Should be at least one ldap server")
+        return {
+            'ldap-server': "ldap://{}".format(ldap_ips[0]),
+            'ldap-user': 'cn=admin,dc=test,dc=com',
+            'ldap-password': 'crapper',
+            'ldap-suffix': 'dc=test,dc=com',
+            'domain-name': 'userdomain',
+            'ldap-query-scope': 'one',
+            'ldap-user-objectclass': 'inetOrgPerson',
+            'ldap-user-id-attribute': 'cn',
+            'ldap-user-name-attribute': 'sn',
+            'ldap-user-enabled-attribute': 'enabled',
+            'ldap-user-enabled-invert': False,
+            'ldap-user-enabled-mask': 0,
+            'ldap-user-enabled-default': 'True',
+            'ldap-group-tree-dn': 'ou=groups,dc=test,dc=com',
+            'ldap-group-objectclass': '',
+            'ldap-group-id-attribute': 'cn',
+            'ldap-group-name-attribute': 'cn',
+            'ldap-group-member-attribute': 'memberUid',
+            'ldap-group-members-are-ids': True,
+            'ldap-config-flags': '{group_objectclass: "posixGroup",'
+                                 ' use_pool: True,'
+                                 ' group_tree_dn: "group_tree_dn_foobar"}',
+        }
+
+    def test_200_config_flags_precedence(self):
+        """Validates precedence when the same config options are used."""
+        application_name = 'keystone-ldap'
+        intended_cfg = self._get_ldap_config()
+        current_cfg, non_string_cfg = (
+            self.config_current_separate_non_string_type_keys(
+                self.non_string_type_keys, intended_cfg, application_name)
+        )
+
+        with self.config_change(
+                {},
+                non_string_cfg,
+                application_name=application_name,
+                reset_to_charm_default=True):
+            with self.config_change(
+                    current_cfg,
+                    intended_cfg,
+                    application_name=application_name):
+                logging.info(
+                    'Performing LDAP settings validation in keystone.conf...'
+                )
+                test_config = lifecycle_utils.get_charm_config(fatal=False)
+                zaza.model.wait_for_application_states(
+                    states=test_config.get("target_deploy_status", {})
+                )
+                units = zaza.model.get_units("keystone-ldap",
+                                             model_name=self.model_name)
+                result = zaza.model.run_on_unit(
+                    units[0].name,
+                    "cat /etc/keystone/domains/keystone.userdomain.conf")
+                # not present in charm config, but present in config flags
+                self.assertIn("use_pool = True", result['stdout'],
+                              "use_pool value is expected to be present and "
+                              "set to True in the config file")
+                # ldap-config-flags overriding empty charm config value
+                self.assertIn("group_objectclass = posixGroup",
+                              result['stdout'],
+                              "group_objectclass is expected to be present and"
+                              " set to posixGroup in the config file")
+                # overridden by charm config, not written to file
+                self.assertNotIn(
+                    "group_tree_dn_foobar",
+                    result['stdout'],
+                    "user_tree_dn ldap-config-flags value needs to be "
+                    "overridden by ldap-user-tree-dn in config file")
+                # complementing the above, value used is from charm setting
+                self.assertIn("group_tree_dn = ou=groups", result['stdout'],
+                              "user_tree_dn value is expected to be present "
+                              "and set to dc=test,dc=com in the config file")

zaza/openstack/charm_tests/magpie/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Collection of code for setting up and testing Magpie."""

zaza/openstack/charm_tests/magpie/tests.py

@@ -0,0 +1,82 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Encapsulate Magpie testing."""
+
+import logging
+
+import zaza
+
+import zaza.model
+import zaza.openstack.charm_tests.test_utils as test_utils
+
+
+class MagpieTest(test_utils.BaseCharmTest):
+    """Base Magpie tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for Magpie charm operation tests."""
+        super(MagpieTest, cls).setUpClass()
+        unit_names = sorted(
+            [i.entity_id
+             for i in zaza.model.get_units('magpie')])
+        cls.test_unit_0 = unit_names[0]
+        cls.test_unit_1 = unit_names[1]
+
+    def test_break_dns_single(self):
+        """Check DNS failure is reflected in workload status."""
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            'mv /etc/resolv.conf /etc/resolv.conf.bak')
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            './hooks/update-status')
+        zaza.model.block_until_unit_wl_message_match(
+            self.test_unit_0,
+            '.*rev dns failed.*')
+        logging.info('Restoring /etc/resolv.conf')
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            'mv /etc/resolv.conf.bak /etc/resolv.conf')
+        logging.info('Updating status')
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            './hooks/update-status')
+
+    def test_break_ping_single(self):
+        """Check ping failure is reflected in workload status."""
+        icmp = "iptables {} INPUT -p icmp --icmp-type echo-request -j REJECT"
+        logging.info('Blocking ping on {}'.format(self.test_unit_1))
+        zaza.model.run_on_unit(
+            self.test_unit_1,
+            icmp.format('--append'))
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            './hooks/update-status')
+        logging.info('Checking status on {}'.format(self.test_unit_0))
+        zaza.model.block_until_unit_wl_message_match(
+            self.test_unit_0,
+            '.*icmp failed.*')
+        logging.info('Allowing ping on {}'.format(self.test_unit_1))
+        zaza.model.run_on_unit(
+            self.test_unit_1,
+            icmp.format('--delete'))
+        zaza.model.run_on_unit(
+            self.test_unit_0,
+            './hooks/update-status')
+        logging.info('Checking status on {}'.format(self.test_unit_0))
+        zaza.model.block_until_unit_wl_message_match(
+            self.test_unit_0,
+            '.*icmp ok.*')

zaza/openstack/charm_tests/manila/tests.py

@@ -16,11 +16,47 @@
 
 """Encapsulate Manila testing."""
 
+import logging
 import tenacity
 
 from manilaclient import client as manilaclient
 
+import zaza.model
+import zaza.openstack.configure.guest as guest
+import zaza.openstack.utilities.openstack as openstack_utils
 import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.openstack.charm_tests.nova.utils as nova_utils
+import zaza.openstack.charm_tests.neutron.tests as neutron_tests
+
+
+def verify_status(stdin, stdout, stderr):
+    """Callable to verify the command output.
+
+    It checks if the command successfully executed.
+
+    This is meant to be given as parameter 'verify' to the helper function
+    'openstack_utils.ssh_command'.
+    """
+    status = stdout.channel.recv_exit_status()
+    if status:
+        logging.info("{}".format(stderr.readlines()[0].strip()))
+    assert status == 0
+
+
+def verify_manila_testing_file(stdin, stdout, stderr):
+    """Callable to verify the command output.
+
+    It checks if the command successfully executed, and it validates the
+    testing file written on the Manila share.
+
+    This is meant to be given as parameter 'verify' to the helper function
+    'openstack_utils.ssh_command'.
+    """
+    verify_status(stdin, stdout, stderr)
+    out = ""
+    for line in iter(stdout.readline, ""):
+        out += line
+    assert out == "test\n"
 
 
 class ManilaTests(test_utils.OpenStackBaseTest):
@@ -35,6 +71,12 @@ class ManilaTests(test_utils.OpenStackBaseTest):
 
     def test_manila_api(self):
         """Test that the Manila API is working."""
+        # The manila charm contains a 'band-aid' for Bug #1706699 which relies
+        # on update-status to bring up services if needed. When the tests run
+        # an update-status hook might not have run so services may still be
+        # stopped so force a hook execution.
+        for unit in zaza.model.get_units('manila'):
+            zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
         self.assertEqual([], self._list_shares())
 
     @tenacity.retry(
@@ -42,3 +84,232 @@ class ManilaTests(test_utils.OpenStackBaseTest):
         wait=tenacity.wait_exponential(multiplier=3, min=2, max=10))
     def _list_shares(self):
         return self.manila_client.shares.list()
+
+
+class ManilaBaseTest(test_utils.OpenStackBaseTest):
+    """Encapsulate a Manila basic functionality test."""
+
+    RESOURCE_PREFIX = 'zaza-manilatests'
+    INSTANCE_KEY = 'bionic'
+    INSTANCE_USERDATA = """#cloud-config
+packages:
+- nfs-common
+"""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running tests."""
+        super(ManilaBaseTest, cls).setUpClass()
+        cls.nova_client = openstack_utils.get_nova_session_client(
+            session=cls.keystone_session)
+        cls.manila_client = manilaclient.Client(
+            session=cls.keystone_session, client_version='2')
+        cls.share_name = 'test-manila-share'
+        cls.share_type_name = 'default_share_type'
+        cls.share_protocol = 'nfs'
+        cls.mount_dir = '/mnt/manila_share'
+        cls.share_network = None
+
+    @classmethod
+    def tearDownClass(cls):
+        """Run class teardown after tests finished."""
+        # Cleanup Nova servers
+        logging.info('Cleaning up test Nova servers')
+        fips_reservations = []
+        for vm in cls.nova_client.servers.list():
+            fips_reservations += neutron_tests.floating_ips_from_instance(vm)
+            vm.delete()
+            openstack_utils.resource_removed(
+                cls.nova_client.servers,
+                vm.id,
+                msg="Waiting for the Nova VM {} to be deleted".format(vm.name))
+
+        # Delete FiPs reservations
+        logging.info('Cleaning up test FiPs reservations')
+        neutron = openstack_utils.get_neutron_session_client(
+            session=cls.keystone_session)
+        for fip in neutron.list_floatingips()['floatingips']:
+            if fip['floating_ip_address'] in fips_reservations:
+                neutron.delete_floatingip(fip['id'])
+
+        # Cleanup Manila shares
+        logging.info('Cleaning up test shares')
+        for share in cls.manila_client.shares.list():
+            share.delete()
+            openstack_utils.resource_removed(
+                cls.manila_client.shares,
+                share.id,
+                msg="Waiting for the Manila share {} to be deleted".format(
+                    share.name))
+
+        # Cleanup test Manila share servers (spawned by the driver when DHSS
+        # is enabled).
+        logging.info('Cleaning up test shares servers (if found)')
+        for server in cls.manila_client.share_servers.list():
+            server.delete()
+            openstack_utils.resource_removed(
+                cls.manila_client.share_servers,
+                server.id,
+                msg="Waiting for the share server {} to be deleted".format(
+                    server.id))
+
+    def _get_mount_options(self):
+        """Get the appropriate mount options used to mount the Manila share.
+
+        :returns: The proper mount options flags for the share protocol.
+        :rtype: string
+        """
+        if self.share_protocol == 'nfs':
+            return 'nfsvers=4.1,proto=tcp'
+        else:
+            raise NotImplementedError(
+                'Share protocol not supported yet: {}'.format(
+                    self.share_protocol))
+
+    def _mount_share_on_instance(self, instance_ip, ssh_user_name,
+                                 ssh_private_key, share_path):
+        """Mount a share into a Nova instance.
+
+        The mount command is executed via SSH.
+
+        :param instance_ip: IP of the Nova instance.
+        :type instance_ip: string
+        :param ssh_user_name: SSH user name.
+        :type ssh_user_name: string
+        :param ssh_private_key: SSH private key.
+        :type ssh_private_key: string
+        :param share_path: Share network path.
+        :type share_path: string
+        """
+        ssh_cmd = (
+            'sudo mkdir -p {0} && '
+            'sudo mount -t {1} -o {2} {3} {0}'.format(
+                self.mount_dir,
+                self.share_protocol,
+                self._get_mount_options(),
+                share_path))
+
+        for attempt in tenacity.Retrying(
+                stop=tenacity.stop_after_attempt(5),
+                wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)):
+            with attempt:
+                openstack_utils.ssh_command(
+                    vm_name="instance-{}".format(instance_ip),
+                    ip=instance_ip,
+                    username=ssh_user_name,
+                    privkey=ssh_private_key,
+                    command=ssh_cmd,
+                    verify=verify_status)
+
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(5),
+        wait=tenacity.wait_exponential(multiplier=3, min=2, max=10))
+    def _write_testing_file_on_instance(self, instance_ip, ssh_user_name,
+                                        ssh_private_key):
+        """Write a file on a Manila share mounted into a Nova instance.
+
+        Write a testing file into the already mounted Manila share from the
+        given Nova instance (which is meant to be validated from another
+        instance). These commands are executed via SSH.
+
+        :param instance_ip: IP of the Nova instance.
+        :type instance_ip: string
+        :param ssh_user_name: SSH user name.
+        :type ssh_user_name: string
+        :param ssh_private_key: SSH private key.
+        :type ssh_private_key: string
+        """
+        openstack_utils.ssh_command(
+            vm_name="instance-{}".format(instance_ip),
+            ip=instance_ip,
+            username=ssh_user_name,
+            privkey=ssh_private_key,
+            command='echo "test" | sudo tee {}/test'.format(
+                self.mount_dir),
+            verify=verify_status)
+
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(5),
+        wait=tenacity.wait_exponential(multiplier=3, min=2, max=10))
+    def _validate_testing_file_from_instance(self, instance_ip, ssh_user_name,
+                                             ssh_private_key):
+        """Validate a file from the Manila share mounted into a Nova instance.
+
+        This is meant to run after the testing file was already written into
+        another Nova instance. It validates the written file. The commands are
+        executed via SSH.
+
+        :param instance_ip: IP of the Nova instance.
+        :type instance_ip: string
+        :param ssh_user_name: SSH user name.
+        :type ssh_user_name: string
+        :param ssh_private_key: SSH private key.
+        :type ssh_private_key: string
+        """
+        openstack_utils.ssh_command(
+            vm_name="instance-{}".format(instance_ip),
+            ip=instance_ip,
+            username=ssh_user_name,
+            privkey=ssh_private_key,
+            command='sudo cat {}/test'.format(self.mount_dir),
+            verify=verify_manila_testing_file)
+
+    def test_manila_share(self):
+        """Test that a Manila share can be accessed on two instances.
+
+        1. Create a share
+        2. Spawn two servers
+        3. Mount it on both
+        4. Write a file on one
+        5. Read it on the other
+        6. Profit
+        """
+        # Create a share
+        share = self.manila_client.shares.create(
+            share_type=self.share_type_name,
+            name=self.share_name,
+            share_proto=self.share_protocol,
+            share_network=self.share_network,
+            size=1)
+
+        # Spawn Servers
+        instance_1 = self.launch_guest(
+            guest_name='ins-1',
+            userdata=self.INSTANCE_USERDATA,
+            instance_key=self.INSTANCE_KEY)
+        instance_2 = self.launch_guest(
+            guest_name='ins-2',
+            userdata=self.INSTANCE_USERDATA,
+            instance_key=self.INSTANCE_KEY)
+
+        fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0]
+        fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0]
+
+        # Wait for the created share to become available before it gets used.
+        openstack_utils.resource_reaches_status(
+            self.manila_client.shares,
+            share.id,
+            wait_iteration_max_time=120,
+            stop_after_attempt=2,
+            expected_status="available",
+            msg="Waiting for a share to become available")
+
+        # Grant access to the Manila share for both Nova instances.
+        share.allow(access_type='ip', access=fip_1, access_level='rw')
+        share.allow(access_type='ip', access=fip_2, access_level='rw')
+
+        ssh_user_name = guest.boot_tests[self.INSTANCE_KEY]['username']
+        privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)
+        share_path = share.export_locations[0]
+
+        # Write a testing file on instance #1
+        self._mount_share_on_instance(
+            fip_1, ssh_user_name, privkey, share_path)
+        self._write_testing_file_on_instance(
+            fip_1, ssh_user_name, privkey)
+
+        # Validate the testing file from instance #2
+        self._mount_share_on_instance(
+            fip_2, ssh_user_name, privkey, share_path)
+        self._validate_testing_file_from_instance(
+            fip_2, ssh_user_name, privkey)

zaza/openstack/charm_tests/manila_ganesha/setup.py

@@ -23,6 +23,9 @@ import zaza.openstack.utilities.openstack as openstack_utils
 from manilaclient import client as manilaclient
 
 
+MANILA_GANESHA_TYPE_NAME = "cephfsnfstype"
+
+
 def setup_ganesha_share_type(manila_client=None):
     """Create a share type for manila with Ganesha.
 
@@ -35,7 +38,7 @@ def setup_ganesha_share_type(manila_client=None):
             session=keystone_session, client_version='2')
 
     manila_client.share_types.create(
-        name="cephfsnfstype", spec_driver_handles_share_servers=False,
+        name=MANILA_GANESHA_TYPE_NAME, spec_driver_handles_share_servers=False,
         extra_specs={
             'vendor_name': 'Ceph',
             'storage_protocol': 'NFS',

zaza/openstack/charm_tests/manila_ganesha/tests.py

@@ -16,116 +16,20 @@
 
 """Encapsulate Manila Ganesha testing."""
 
-from tenacity import Retrying, stop_after_attempt, wait_exponential
+from zaza.openstack.charm_tests.manila_ganesha.setup import (
+    MANILA_GANESHA_TYPE_NAME,
+)
 
-from manilaclient import client as manilaclient
-
-import zaza.openstack.charm_tests.neutron.tests as neutron_tests
-import zaza.openstack.charm_tests.nova.utils as nova_utils
-import zaza.openstack.charm_tests.test_utils as test_utils
-import zaza.openstack.configure.guest as guest
-import zaza.openstack.utilities.openstack as openstack_utils
+import zaza.openstack.charm_tests.manila.tests as manila_tests
 
 
-class ManilaGaneshaTests(test_utils.OpenStackBaseTest):
+class ManilaGaneshaTests(manila_tests.ManilaBaseTest):
     """Encapsulate Manila Ganesha tests."""
 
-    RESOURCE_PREFIX = 'zaza-manilatests'
-    INSTANCE_USERDATA = """#cloud-config
-packages:
-- nfs-common
-"""
-
     @classmethod
     def setUpClass(cls):
         """Run class setup for running tests."""
         super(ManilaGaneshaTests, cls).setUpClass()
-        cls.nova_client = (
-            openstack_utils.get_nova_session_client(cls.keystone_session))
-        cls.manila_client = manilaclient.Client(
-            session=cls.keystone_session, client_version='2')
-
-    def test_manila_share(self):
-        """Test that Manila + Ganesha shares can be accessed on two instances.
-
-        1. create a share
-        2. Spawn two servers
-        3. mount it on both
-        4. write a file on one
-        5. read it on the other
-        6. profit
-        """
-        # Create a share
-        share = self.manila_client.shares.create(
-            share_type='cephfsnfstype', name='cephnfsshare1',
-            share_proto="nfs", size=1)
-
-        # Spawn Servers
-        instance_1, instance_2 = self.launch_guests(
-            userdata=self.INSTANCE_USERDATA)
-
-        fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0]
-        fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0]
-
-        # Wait for the created share to become available before it gets used.
-        openstack_utils.resource_reaches_status(
-            self.manila_client.shares,
-            share.id,
-            wait_iteration_max_time=120,
-            stop_after_attempt=2,
-            expected_status="available",
-            msg="Waiting for a share to become available")
-
-        share.allow(access_type='ip', access=fip_1, access_level='rw')
-        share.allow(access_type='ip', access=fip_2, access_level='rw')
-
-        # Mount Share
-        username = guest.boot_tests['bionic']['username']
-        password = guest.boot_tests['bionic'].get('password')
-        privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)
-
-        # Write a file on instance_1
-        def verify_setup(stdin, stdout, stderr):
-            status = stdout.channel.recv_exit_status()
-            self.assertEqual(status, 0)
-
-        mount_path = share.export_locations[0]
-
-        for attempt in Retrying(
-                stop=stop_after_attempt(3),
-                wait=wait_exponential(multiplier=1, min=2, max=10)):
-            with attempt:
-                openstack_utils.ssh_command(
-                    username, fip_1, 'instance-1',
-                    'sudo mkdir -p /mnt/ceph && '
-                    'sudo mount -t nfs -o nfsvers=4.1,proto=tcp '
-                    '{} /mnt/ceph && '
-                    'echo "test" | sudo tee /mnt/ceph/test'.format(
-                        mount_path),
-                    password=password, privkey=privkey, verify=verify_setup)
-
-        for attempt in Retrying(
-                stop=stop_after_attempt(3),
-                wait=wait_exponential(multiplier=1, min=2, max=10)):
-            with attempt:
-                # Setup that file on instance_2
-                openstack_utils.ssh_command(
-                    username, fip_2, 'instance-2',
-                    'sudo mkdir -p /mnt/ceph && '
-                    'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp '
-                    '{} /mnt/ceph'
-                    .format(mount_path),
-                    password=password, privkey=privkey, verify=verify_setup)
-
-        def verify(stdin, stdout, stderr):
-            status = stdout.channel.recv_exit_status()
-            self.assertEqual(status, 0)
-            out = ""
-            for line in iter(stdout.readline, ""):
-                out += line
-            self.assertEqual(out, "test\n")
-
-        openstack_utils.ssh_command(
-            username, fip_2, 'instance-2',
-            'sudo cat /mnt/ceph/test',
-            password=password, privkey=privkey, verify=verify)
+        cls.share_name = 'cephnfsshare1'
+        cls.share_type_name = MANILA_GANESHA_TYPE_NAME
+        cls.share_protocol = 'nfs'

zaza/openstack/charm_tests/manila_netapp/__init__.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Encapsulate Manila NetApp setup and testing."""

zaza/openstack/charm_tests/manila_netapp/setup.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Encapsulate Manila NetApp setup."""
+
+import zaza.openstack.utilities.openstack as openstack_utils
+import zaza.openstack.charm_tests.neutron.setup as neutron_setup
+
+
+MANILA_NETAPP_TYPE_NAME = "netapp-ontap"
+MANILA_NETAPP_BACKEND_NAME = "netapp-ontap"
+
+MANILA_NETAPP_DHSS_TYPE_NAME = "netapp-ontap-dhss"
+MANILA_NETAPP_DHSS_BACKEND_NAME = "netapp-ontap-dhss"
+
+MANILA_NETAPP_SHARE_NET_NAME = "netapp-ontap-share-network"
+
+
+def create_netapp_share_type(manila_client=None):
+    """Create a share type for Manila with NetApp Data ONTAP driver.
+
+    :param manila_client: Authenticated manilaclient
+    :type manila_client: manilaclient.Client
+    """
+    if manila_client is None:
+        manila_client = openstack_utils.get_manila_session_client(
+            openstack_utils.get_overcloud_keystone_session())
+
+    manila_client.share_types.create(
+        name=MANILA_NETAPP_TYPE_NAME,
+        spec_driver_handles_share_servers=False,
+        extra_specs={
+            'vendor_name': 'NetApp',
+            'share_backend_name': MANILA_NETAPP_BACKEND_NAME,
+            'storage_protocol': 'NFS_CIFS',
+        })
+
+
+def create_netapp_dhss_share_type(manila_client=None):
+    """Create a DHSS share type for Manila with NetApp Data ONTAP driver.
+
+    :param manila_client: Authenticated manilaclient
+    :type manila_client: manilaclient.Client
+    """
+    if manila_client is None:
+        manila_client = openstack_utils.get_manila_session_client(
+            openstack_utils.get_overcloud_keystone_session())
+
+    manila_client.share_types.create(
+        name=MANILA_NETAPP_DHSS_TYPE_NAME,
+        spec_driver_handles_share_servers=True,
+        extra_specs={
+            'vendor_name': 'NetApp',
+            'share_backend_name': MANILA_NETAPP_DHSS_BACKEND_NAME,
+            'storage_protocol': 'NFS_CIFS',
+        })
+
+
+def create_netapp_share_network(manila_client=None):
+    """Create a Manila share network from the existing provider network.
+
+    This setup function assumes that 'neutron.setup.basic_overcloud_network'
+    is called to have the proper tenant networks setup.
+
+    The share network will be bound to the provider network configured by
+    'neutron.setup.basic_overcloud_network'.
+    """
+    session = openstack_utils.get_overcloud_keystone_session()
+    if manila_client is None:
+        manila_client = openstack_utils.get_manila_session_client(session)
+
+    neutron = openstack_utils.get_neutron_session_client(session)
+    external_net = neutron.find_resource(
+        'network',
+        neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_net_name'])
+    external_subnet = neutron.find_resource(
+        'subnet',
+        neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_subnet_name'])
+
+    manila_client.share_networks.create(
+        name=MANILA_NETAPP_SHARE_NET_NAME,
+        neutron_net_id=external_net['id'],
+        neutron_subnet_id=external_subnet['id'])

zaza/openstack/charm_tests/manila_netapp/tests.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Encapsulate Manila NetApp testing."""
+
+from zaza.openstack.charm_tests.manila_netapp.setup import (
+    MANILA_NETAPP_TYPE_NAME,
+    MANILA_NETAPP_DHSS_TYPE_NAME,
+    MANILA_NETAPP_SHARE_NET_NAME,
+)
+
+import zaza.openstack.charm_tests.manila.tests as manila_tests
+
+
+class ManilaNetAppNFSTest(manila_tests.ManilaBaseTest):
+    """Encapsulate Manila NetApp NFS test."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running tests."""
+        super(ManilaNetAppNFSTest, cls).setUpClass()
+        cls.share_name = 'netapp-ontap-share'
+        cls.share_type_name = MANILA_NETAPP_TYPE_NAME
+        cls.share_protocol = 'nfs'
+
+
+class ManilaNetAppDHSSNFSTest(manila_tests.ManilaBaseTest):
+    """Encapsulate Manila NetApp NFS test."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running tests."""
+        super(ManilaNetAppDHSSNFSTest, cls).setUpClass()
+        cls.share_name = 'netapp-ontap-dhss-share'
+        cls.share_type_name = MANILA_NETAPP_DHSS_TYPE_NAME
+        cls.share_protocol = 'nfs'
+        cls.share_network = cls.manila_client.share_networks.find(
+            name=MANILA_NETAPP_SHARE_NET_NAME)

zaza/openstack/charm_tests/mysql/tests.py

@@ -113,6 +113,56 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest):
             if _primary_ip in unit.public_address:
                 return unit
 
+    def get_blocked_mysql_routers(self):
+        """Get blocked mysql routers.
+
+        :returns: List of blocked mysql-router unit names
+        :rtype: List[str]
+        """
+        # Make sure mysql-router units are up to date
+        # We cannot assume they are as there is up to a five minute delay
+        mysql_router_units = []
+        for application in self.get_applications_with_substring_in_name(
+                "mysql-router"):
+            for unit in zaza.model.get_units(application):
+                mysql_router_units.append(unit.entity_id)
+        self.run_update_status_hooks(mysql_router_units)
+
+        # Get up to date status
+        status = zaza.model.get_status().applications
+        blocked_mysql_routers = []
+        # Check if the units are blocked
+        for application in self.get_applications_with_substring_in_name(
+                "mysql-router"):
+            # Subordinate dance with primary
+            # There is no satus[applicatoin]["units"] for subordinates
+            _subordinate_to = status[application].subordinate_to[0]
+            for appunit in status[_subordinate_to].units:
+                for subunit in (
+                        status[_subordinate_to].
+                        units[appunit].subordinates.keys()):
+                    if "blocked" in (
+                            status[_subordinate_to].units[appunit].
+                            subordinates[subunit].workload_status.status):
+                        blocked_mysql_routers.append(subunit)
+        return blocked_mysql_routers
+
+    def restart_blocked_mysql_routers(self):
+        """Restart blocked mysql routers.
+
+        :returns: None
+        :rtype: None
+        """
+        # Check for blocked mysql-router units
+        blocked_mysql_routers = self.get_blocked_mysql_routers()
+        for unit in blocked_mysql_routers:
+            logging.warning(
+                "Restarting blocked mysql-router unit {}"
+                .format(unit))
+            zaza.model.run_on_unit(
+                unit,
+                "systemctl restart {}".format(unit.rpartition("/")[0]))
+
 
 class MySQLCommonTests(MySQLBaseTest):
     """Common mysql charm tests."""
@@ -169,6 +219,15 @@ class MySQLCommonTests(MySQLBaseTest):
         """
         with self.pause_resume(self.services):
             logging.info("Testing pause resume")
+
+        logging.info("Wait till model is idle ...")
+        zaza.model.block_until_all_units_idle()
+
+        # If there are any blocekd mysql routers restart them.
+        self.restart_blocked_mysql_routers()
+        assert not self.get_blocked_mysql_routers(), (
+            "Should no longer be blocked mysql-router units")
+
         logging.info("Passed pause and resume test.")
 
 
@@ -589,6 +648,10 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest):
                 unit.entity_id,
                 'blocked')
 
+        # Wait until update-status hooks have completed
+        logging.info("Wait till model is idle ...")
+        zaza.model.block_until_all_units_idle()
+
         logging.info("Execute reboot-cluster-from-complete-outage "
                      "action after cold boot ...")
         # We do not know which unit has the most up to date data
@@ -753,13 +816,22 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest):
         logging.info("Scale in test: remove leader")
         leader, nons = self.get_leaders_and_non_leaders()
         leader_unit = zaza.model.get_unit_from_name(leader)
-        zaza.model.destroy_unit(self.application_name, leader)
-
-        logging.info("Wait until unit is in waiting state ...")
-        zaza.model.block_until_unit_wl_status(nons[0], "waiting")
 
+        # Wait until we are idle in the hopes clients are not running
+        # update-status hooks
         logging.info("Wait till model is idle ...")
         zaza.model.block_until_all_units_idle()
+        zaza.model.destroy_unit(self.application_name, leader)
+
+        logging.info("Wait until all only 2 units ...")
+        zaza.model.block_until_unit_count(self.application, 2)
+
+        logging.info("Wait until all units are cluster incomplete ...")
+        zaza.model.block_until_wl_status_info_starts_with(
+            self.application, "'cluster' incomplete")
+
+        # Show status
+        logging.info(self.get_cluster_status())
 
         logging.info(
             "Removing old unit from cluster: {} "
@@ -786,6 +858,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest):
         logging.info("Adding unit after removed unit ...")
         zaza.model.add_unit(self.application_name)
 
+        logging.info("Wait until 3 units ...")
+        zaza.model.block_until_unit_count(self.application, 3)
+
         logging.info("Wait for application states ...")
         zaza.model.wait_for_application_states(states=self.states)
 
@@ -801,6 +876,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest):
         logging.info("Adding unit after full cluster ...")
         zaza.model.add_unit(self.application_name)
 
+        logging.info("Wait until 4 units ...")
+        zaza.model.block_until_unit_count(self.application, 4)
+
         logging.info("Wait for application states ...")
         zaza.model.wait_for_application_states(states=self.states)
 
@@ -810,19 +888,26 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest):
         We start with a four node full cluster, remove one, down to a three
         node full cluster.
         """
-        logging.info("Wait till model is idle ...")
-        zaza.model.block_until_all_units_idle()
-
         leader, nons = self.get_leaders_and_non_leaders()
         non_leader_unit = zaza.model.get_unit_from_name(nons[0])
-        zaza.model.destroy_unit(self.application_name, nons[0])
 
+        # Wait until we are idle in the hopes clients are not running
+        # update-status hooks
         logging.info("Wait till model is idle ...")
         zaza.model.block_until_all_units_idle()
 
+        zaza.model.destroy_unit(self.application_name, nons[0])
+
         logging.info("Scale in test: back down to three")
+        logging.info("Wait until 3 units ...")
+        zaza.model.block_until_unit_count(self.application, 3)
+
+        logging.info("Wait for status ready ...")
         zaza.model.wait_for_application_states(states=self.states)
 
+        # Show status
+        logging.info(self.get_cluster_status())
+
         logging.info(
             "Removing old unit from cluster: {} "
             .format(non_leader_unit.public_address))
@@ -835,3 +920,109 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest):
         assert action.data.get("results") is not None, (
             "Remove instance action failed: No results: {}"
             .format(action.data))
+
+
+class MySQLInnoDBClusterPartitionTest(MySQLBaseTest):
+    """MySQL partition handling."""
+
+    def test_850_force_quorum_using_partition_of(self):
+        """Force quorum using partition of instance with given address.
+
+        After outage, cluster can end up without quorum. Force it.
+        """
+        logging.info("Wait till model is idle ...")
+        zaza.model.block_until_all_units_idle()
+
+        # Block all traffic across mysql instances: 0<-1, 1<-2 and 2<-0
+        mysql_units = [unit for unit in zaza.model.get_units(self.application)]
+        no_of_units = len(mysql_units)
+        for index, unit in enumerate(mysql_units):
+            next_unit = mysql_units[(index+1) % no_of_units]
+            ip_address = next_unit.public_address
+            cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address)
+            zaza.model.async_run_on_unit(unit, cmd)
+
+        logging.info(
+            "Wait till all {} units are in state 'blocked' ..."
+            .format(self.application))
+        for unit in zaza.model.get_units(self.application):
+            zaza.model.block_until_unit_wl_status(
+                unit.entity_id,
+                'blocked',
+                negate_match=True)
+
+        logging.info("Wait till model is idle ...")
+        zaza.model.block_until_all_units_idle()
+
+        logging.info("Execute force-quorum-using-partition-of action ...")
+
+        # Select "quorum leader" unit
+        leader_unit = mysql_units[0]
+        action = zaza.model.run_action(
+            leader_unit.entity_id,
+            "force-quorum-using-partition-of",
+            action_params={
+                "address": leader_unit.public_address,
+                'i-really-mean-it': True
+            })
+
+        assert action.data.get("results") is not None, (
+            "Force quorum using partition of action failed: {}"
+            .format(action.data))
+        logging.debug(
+            "Results from running 'force-quorum' command ...\n{}".format(
+                action.data))
+
+        logging.info("Wait till model is idle ...")
+        try:
+            zaza.model.block_until_all_units_idle()
+        except zaza.model.UnitError:
+            self.resolve_update_status_errors()
+            zaza.model.block_until_all_units_idle()
+
+        # Unblock all traffic across mysql instances
+        for unit in zaza.model.get_units(self.application):
+            cmd = "sudo iptables -F"
+            zaza.model.async_run_on_unit(unit, cmd)
+
+        logging.info("Wait for application states ...")
+        for unit in zaza.model.get_units(self.application):
+            zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
+        test_config = lifecycle_utils.get_charm_config(fatal=False)
+        zaza.model.wait_for_application_states(
+            states=test_config.get("target_deploy_status", {}))
+
+
+class MySQLRouterTests(test_utils.OpenStackBaseTest):
+    """MySQL Router Tests."""
+
+    @classmethod
+    def setUpClass(cls, application_name="keystone-mysql-router"):
+        """Run class setup for running mysql-router tests."""
+        super().setUpClass(application_name=application_name)
+        cls.application = application_name
+        cls.services = ["mysqlrouter"]
+        # Config file affected by juju set config change
+        cls.conf_file = (
+            "/var/lib/mysql/{}-mysql-router/mysqlrouter.conf"
+            .format(application_name))
+
+    def test_910_restart_on_config_change(self):
+        """Checking restart happens on config change.
+
+        Change max connections and assert that change propagates to the correct
+        file and that services are restarted as a result
+        """
+        # Expected default and alternate values
+        set_default = {"ttl": ".5"}
+        set_alternate = {"ttl": "7"}
+
+        # Make config change, check for service restarts
+        logging.info("Setting TTL ...")
+        self.restart_on_changed(
+            self.conf_file,
+            set_default,
+            set_alternate,
+            {}, {},
+            self.services)
+        logging.info("Passed restart on changed test.")

zaza/openstack/charm_tests/mysql/utils.py

@@ -19,8 +19,16 @@ import zaza.model as model
 
 async def complete_cluster_series_upgrade():
     """Run the complete-cluster-series-upgrade action on the lead unit."""
-    # TODO: Make this work across either mysql or percona-cluster names
-    await model.async_run_action_on_leader(
-        'mysql',
-        'complete-cluster-series-upgrade',
-        action_params={})
+    # Note that some models use mysql as the application name, and other's use
+    # percona-cluster.  Try mysql first, and if it doesn't exist, then try
+    # percona-cluster instead.
+    try:
+        await model.async_run_action_on_leader(
+            'mysql',
+            'complete-cluster-series-upgrade',
+            action_params={})
+    except KeyError:
+        await model.async_run_action_on_leader(
+            'percona-cluster',
+            'complete-cluster-series-upgrade',
+            action_params={})

zaza/openstack/charm_tests/neutron/setup.py

@@ -15,6 +15,7 @@
 """Setup for Neutron deployments."""
 
 import functools
+import logging
 
 from zaza.openstack.configure import (
     network,
@@ -89,12 +90,25 @@ def basic_overcloud_network(limit_gws=None):
         'configure_gateway_ext_port_use_juju_wait', True)
 
     # Handle network for OpenStack-on-OpenStack scenarios
-    if juju_utils.get_provider_type() == "openstack":
+    provider_type = juju_utils.get_provider_type()
+    if provider_type == "openstack":
         undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session()
         network.setup_gateway_ext_port(network_config,
                                        keystone_session=undercloud_ks_sess,
-                                       limit_gws=None,
+                                       limit_gws=limit_gws,
                                        use_juju_wait=use_juju_wait)
+    elif provider_type == "maas":
+        # NOTE(fnordahl): After validation of the MAAS+Netplan Open vSwitch
+        # integration support, we would most likely want to add multiple modes
+        # of operation with MAAS.
+        #
+        # Perform charm based OVS configuration
+        openstack_utils.configure_charmed_openstack_on_maas(
+            network_config, limit_gws=limit_gws)
+    else:
+        logging.warning('Unknown Juju provider type, "{}", will not perform'
+                        ' charm network configuration.'
+                        .format(provider_type))
 
     # Confugre the overcloud network
     network.setup_sdn(network_config, keystone_session=keystone_session)

zaza/openstack/charm_tests/neutron/tests.py

@@ -592,6 +592,58 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests):
             logging.info('Testing pause resume')
 
 
+class NeutronBridgePortMappingTest(NeutronPluginApiSharedTests):
+    """Test correct handling of network-bridge-port mapping functionality."""
+
+    def test_600_conflict_data_ext_ports(self):
+        """Verify proper handling of conflict between data-port and ext-port.
+
+        Configuring ext-port and data-port at the same time should make the
+        charm to enter "blocked" state. After unsetting ext-port charm should
+        be active again.
+        """
+        if self.application_name not in ["neutron-gateway",
+                                         "neutron-openvswitch"]:
+            logging.debug("Skipping test, charm under test is not "
+                          "neutron-gateway or neutron-openvswitch")
+            return
+
+        current_data_port = zaza.model.get_application_config(
+            self.application_name).get("data-port").get("value", "")
+        current_ext_port = zaza.model.get_application_config(
+            self.application_name).get("ext-port").get("value", "")
+        logging.debug("Current data-port: '{}'".format(current_data_port))
+        logging.debug("Current data-port: '{}'".format(current_ext_port))
+
+        test_config = zaza.charm_lifecycle.utils.get_charm_config(
+            fatal=False)
+        current_state = test_config.get("target_deploy_status", {})
+        blocked_state = copy.deepcopy(current_state)
+        blocked_state[self.application_name] = {
+            "workload-status": "blocked",
+            "workload-status-message":
+                "ext-port set when data-port set: see config.yaml"}
+
+        logging.info("Setting conflicting ext-port and data-port options")
+        zaza.model.set_application_config(
+            self.application_name, {"data-port": "br-phynet43:eth43",
+                                    "ext-port": "br-phynet43:eth43"})
+        zaza.model.wait_for_application_states(states=blocked_state)
+
+        # unset ext-port and wait for app state to return to active
+        logging.info("Unsetting conflicting ext-port option")
+        zaza.model.set_application_config(
+            self.application_name, {"ext-port": ""})
+        zaza.model.wait_for_application_states(states=current_state)
+
+        # restore original config
+        zaza.model.set_application_config(
+            self.application_name, {'data-port': current_data_port,
+                                    'ext-port': current_ext_port})
+        zaza.model.wait_for_application_states(states=current_state)
+        logging.info('OK')
+
+
 class NeutronOvsVsctlTest(NeutronPluginApiSharedTests):
     """Test 'ovs-vsctl'-related functionality on Neutron charms."""
 
@@ -610,7 +662,8 @@ class NeutronOvsVsctlTest(NeutronPluginApiSharedTests):
                         unit.name, bridge_name
                     ) + ' is marked as managed by us'
                 )
-                expected_external_id = 'charm-neutron-gateway=managed'
+                expected_external_id = 'charm-{}=managed'.format(
+                    self.application_name)
                 actual_external_id = zaza.model.run_on_unit(
                     unit.entity_id,
                     'ovs-vsctl br-get-external-id {}'.format(bridge_name),
@@ -952,3 +1005,47 @@ class NeutronNetworkingVRRPTests(NeutronNetworkingBase):
                 uc_neutron_client,
                 gateway_hostname)
             self.check_connectivity(instance_1, instance_2)
+
+
+class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest):
+    """Deferred restart tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for deferred restart tests."""
+        super().setUpClass(application_name='neutron-openvswitch')
+
+    def run_tests(self):
+        """Run deferred restart tests."""
+        # Trigger a config change which triggers a deferred hook.
+        self.run_charm_change_hook_test('config-changed')
+
+        # Trigger a package change which requires a restart
+        self.run_package_change_test(
+            'openvswitch-switch',
+            'openvswitch-switch')
+
+
+class NeutronGatewayDeferredRestartTest(test_utils.BaseDeferredRestartTest):
+    """Deferred restart tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for deferred restart tests."""
+        super().setUpClass(application_name='neutron-gateway')
+
+    def run_tests(self):
+        """Run deferred restart tests."""
+        # Trigger a config change which requires a restart
+        self.run_charm_change_restart_test(
+            'neutron-l3-agent',
+            '/etc/neutron/neutron.conf')
+
+        # Trigger a package change which requires a restart
+        self.run_package_change_test(
+            'openvswitch-switch',
+            'openvswitch-switch')
+
+    def check_clear_hooks(self):
+        """Gateway does not defer hooks so noop."""
+        return

zaza/openstack/charm_tests/nova/setup.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Code for configureing nova."""
+"""Code for configuring nova."""
+
+import tenacity
 
 import zaza.openstack.utilities.openstack as openstack_utils
 from zaza.openstack.utilities import (
@@ -21,6 +23,9 @@ from zaza.openstack.utilities import (
 import zaza.openstack.charm_tests.nova.utils as nova_utils
 
 
+@tenacity.retry(stop=tenacity.stop_after_attempt(3),
+                wait=tenacity.wait_exponential(
+                multiplier=1, min=2, max=10))
 def create_flavors(nova_client=None):
     """Create basic flavors.
 
@@ -43,6 +48,9 @@ def create_flavors(nova_client=None):
                 flavorid=nova_utils.FLAVORS[flavor]['flavorid'])
 
 
+@tenacity.retry(stop=tenacity.stop_after_attempt(3),
+                wait=tenacity.wait_exponential(
+                multiplier=1, min=2, max=10))
 def manage_ssh_key(nova_client=None):
     """Create basic flavors.
 

zaza/openstack/charm_tests/nova/tests.py

@@ -19,6 +19,8 @@
 import json
 import logging
 import unittest
+from configparser import ConfigParser
+from time import sleep
 
 import zaza.model
 import zaza.openstack.charm_tests.glance.setup as glance_setup
@@ -38,37 +40,294 @@ class BaseGuestCreateTest(unittest.TestCase):
         zaza.openstack.configure.guest.launch_instance(instance_key)
 
 
-class CirrosGuestCreateTest(BaseGuestCreateTest):
+class CirrosGuestCreateTest(test_utils.OpenStackBaseTest):
     """Tests to launch a cirros image."""
 
     def test_launch_small_instance(self):
         """Launch a cirros instance and test connectivity."""
-        zaza.openstack.configure.guest.launch_instance(
-            glance_setup.CIRROS_IMAGE_NAME)
+        self.RESOURCE_PREFIX = 'zaza-nova'
+        self.launch_guest(
+            'cirros', instance_key=glance_setup.CIRROS_IMAGE_NAME)
+
+    def tearDown(self):
+        """Cleanup of VM guests."""
+        self.resource_cleanup()
 
 
-class LTSGuestCreateTest(BaseGuestCreateTest):
+class LTSGuestCreateTest(test_utils.OpenStackBaseTest):
     """Tests to launch a LTS image."""
 
     def test_launch_small_instance(self):
         """Launch a Bionic instance and test connectivity."""
-        zaza.openstack.configure.guest.launch_instance(
-            glance_setup.LTS_IMAGE_NAME)
+        self.RESOURCE_PREFIX = 'zaza-nova'
+        self.launch_guest(
+            'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME)
+
+    def tearDown(self):
+        """Cleanup of VM guests."""
+        self.resource_cleanup()
 
 
-class LTSGuestCreateVolumeBackedTest(BaseGuestCreateTest):
+class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest):
     """Tests to launch a LTS image."""
 
     def test_launch_small_instance(self):
         """Launch a Bionic instance and test connectivity."""
-        zaza.openstack.configure.guest.launch_instance(
-            glance_setup.LTS_IMAGE_NAME,
+        self.RESOURCE_PREFIX = 'zaza-nova'
+        self.launch_guest(
+            'volume-backed-ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME,
             use_boot_volume=True)
 
+    def tearDown(self):
+        """Cleanup of VM guests."""
+        self.resource_cleanup()
 
-class NovaCompute(test_utils.OpenStackBaseTest):
+
+class NovaCommonTests(test_utils.OpenStackBaseTest):
+    """nova-compute and nova-cloud-controller common tests."""
+
+    XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka')
+    XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata')
+    XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens')
+    BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens')
+    BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky')
+    BIONIC_TRAIN = openstack_utils.get_os_release('bionic_train')
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running nova-cloud-controller tests."""
+        super(NovaCommonTests, cls).setUpClass()
+        cls.current_release = openstack_utils.get_os_release()
+
+    def _test_pci_alias_config(self, app_name, service_list):
+        logging.info('Checking pci aliases in nova config...')
+
+        # Expected default and alternate values
+        current_value = zaza.model.get_application_config(
+            app_name)['pci-alias']
+        try:
+            current_value = current_value['value']
+        except KeyError:
+            current_value = None
+        new_value = '[{}, {}]'.format(
+            json.dumps({
+                'name': 'IntelNIC',
+                'capability_type': 'pci',
+                'product_id': '1111',
+                'vendor_id': '8086',
+                'device_type': 'type-PF'
+            }, sort_keys=True),
+            json.dumps({
+                'name': ' Cirrus Logic ',
+                'capability_type': 'pci',
+                'product_id': '0ff2',
+                'vendor_id': '10de',
+                'device_type': 'type-PCI'
+            }, sort_keys=True))
+
+        set_default = {'pci-alias': current_value}
+        set_alternate = {'pci-alias': new_value}
+
+        expected_conf_section = 'pci'
+        expected_conf_key = 'alias'
+
+        default_entry = {expected_conf_section: {}}
+        alternate_entry = {expected_conf_section: {
+            expected_conf_key: [
+                ('{"capability_type": "pci", "device_type": "type-PF", '
+                 '"name": "IntelNIC", "product_id": "1111", '
+                 '"vendor_id": "8086"}'),
+                ('{"capability_type": "pci", "device_type": "type-PCI", '
+                 '"name": " Cirrus Logic ", "product_id": "0ff2", '
+                 '"vendor_id": "10de"}')]}}
+
+        # Config file affected by juju set config change
+        conf_file = '/etc/nova/nova.conf'
+
+        # Make config change, check for service restarts
+        logging.info(
+            'Setting config on {} to {}'.format(app_name, set_alternate))
+        self.restart_on_changed(
+            conf_file,
+            set_default,
+            set_alternate,
+            default_entry,
+            alternate_entry,
+            service_list)
+
+
+class CloudActions(test_utils.OpenStackBaseTest):
+    """Test actions from actions/cloud.py."""
+
+    def fetch_nova_service_hostname(self, unit_name):
+        """
+        Fetch hostname used to register with nova-cloud-controller.
+
+        When nova-compute registers with nova-cloud-controller it uses either
+        config variable from '/etc/nova/nova.conf` or host's hostname to
+        identify itself. We need to fetch this value directly from the unit,
+        otherwise it's not possible to correlate entries from
+        `nova service-list` with nova-compute units.
+
+        :param unit_name: nova-compute unit name.
+        :return: hostname used when registering to cloud-controller
+        """
+        nova_cfg = ConfigParser()
+
+        result = zaza.model.run_on_unit(unit_name,
+                                        'cat /etc/nova/nova.conf')
+        nova_cfg.read_string(result['Stdout'])
+
+        try:
+            nova_service_name = nova_cfg['DEFAULT']['host']
+        except KeyError:
+            # Fallback to hostname if 'host' variable is not present in the
+            # config
+            result = zaza.model.run_on_unit(unit_name, 'hostname')
+            nova_service_name = result['Stdout'].rstrip('\n')
+
+        if not nova_service_name:
+            self.fail("Failed to fetch nova service name from"
+                      " nova-compute unit.")
+        return nova_service_name
+
+    def test_940_enable_disable_actions(self):
+        """Test disable/enable actions on nova-compute units."""
+        nova_units = zaza.model.get_units('nova-compute',
+                                          model_name=self.model_name)
+
+        # Check that nova-compute services are enabled before testing
+        for service in self.nova_client.services.list(binary='nova-compute'):
+            self.assertEqual(service.status, 'enabled')
+
+        # Run 'disable' action on units
+        zaza.model.run_action_on_units([unit.name for unit in nova_units],
+                                       'disable')
+
+        # Check action results via nova API
+        for service in self.nova_client.services.list(binary='nova-compute'):
+            self.assertEqual(service.status, 'disabled')
+
+        # Run 'enable' action on units
+        zaza.model.run_action_on_units([unit.name for unit in nova_units],
+                                       'enable')
+
+        # Check action results via nova API
+        for service in self.nova_client.services.list(binary='nova-compute'):
+            self.assertEqual(service.status, 'enabled')
+
+    def test_950_instance_count_action(self):
+        """Test that action 'instance-count' returns expected values."""
+        def check_instance_count(expect_count, unit_name):
+            """Assert that unit with 'unit_name' has 'expect_count' of VMs.
+
+            :param expect_count: How many VMs are expected to be running
+            :param unit_name: Name of the target nova-compute unit
+            :return: None
+            :raises AssertionError: If result of the 'instance-count' action
+                                    does not match 'expect_count'.
+            """
+            logging.debug('Running "instance-count" action on unit "{}".'
+                          'Expecting result: {}'.format(unit_name,
+                                                        expect_count))
+            result = zaza.model.run_action(unit_name, 'instance-count')
+            self.assertEqual(result.status, 'completed')
+            instances = result.data.get('results', {}).get('instance-count')
+            self.assertEqual(instances, str(expect_count))
+
+        nova_unit = zaza.model.get_units('nova-compute',
+                                         model_name=self.model_name)[0]
+
+        check_instance_count(0, nova_unit.entity_id)
+
+        self.RESOURCE_PREFIX = 'zaza-nova'
+        self.launch_guest(
+            'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME)
+
+        check_instance_count(1, nova_unit.entity_id)
+
+        self.resource_cleanup()
+
+    def test_960_remove_from_cloud_actions(self):
+        """Test actions remove-from-cloud and register-to-cloud.
+
+        Note (martin-kalcok): This test requires that nova-compute unit is not
+        running any VMs. If there are any leftover VMs from previous tests,
+        action `remove-from-cloud` will fail.
+        """
+        def wait_for_nova_compute_count(expected_count):
+            """Wait for expected number of nova compute services to be present.
+
+            Returns True or False based on whether the expected number of nova
+            compute services was reached within the timeout. Checks are
+            performed every 10 second in the span of maximum 5 minutes.
+            """
+            sleep_timeout = 1  # don't waste 10 seconds on the first run
+
+            for _ in range(31):
+                sleep(sleep_timeout)
+                service_list = self.nova_client.services.list(
+                    host=service_name, binary='nova-compute')
+                if len(service_list) == expected_count:
+                    return True
+                sleep_timeout = 10
+            return False
+
+        all_units = zaza.model.get_units('nova-compute',
+                                         model_name=self.model_name)
+
+        unit_to_remove = all_units[0]
+
+        service_name = self.fetch_nova_service_hostname(unit_to_remove.name)
+
+        registered_nova_services = self.nova_client.services.list(
+            host=service_name, binary='nova-compute')
+
+        service_count = len(registered_nova_services)
+        if service_count < 1:
+            self.fail("Unit '{}' has no nova-compute services registered in"
+                      " nova-cloud-controller".format(unit_to_remove.name))
+        elif service_count > 1:
+            self.fail("Unexpected number of nova-compute services registered"
+                      " in nova-cloud controller. Expecting: 1, found: "
+                      "{}".format(service_count))
+
+        # run action remove-from-cloud and wait for the results in
+        # nova-cloud-controller
+        zaza.model.run_action_on_units([unit_to_remove.name],
+                                       'remove-from-cloud',
+                                       raise_on_failure=True)
+
+        # Wait for nova-compute service to be removed from the
+        # nova-cloud-controller
+        if not wait_for_nova_compute_count(0):
+            self.fail("nova-compute service was not unregistered from the "
+                      "nova-cloud-controller as expected.")
+
+        # run action register-to-cloud to revert previous action
+        # and wait for the results in nova-cloud-controller
+        zaza.model.run_action_on_units([unit_to_remove.name],
+                                       'register-to-cloud',
+                                       raise_on_failure=True)
+
+        if not wait_for_nova_compute_count(1):
+            self.fail("nova-compute service was not re-registered to the "
+                      "nova-cloud-controller as expected.")
+
+
+class NovaCompute(NovaCommonTests):
     """Run nova-compute specific tests."""
 
+    def test_311_pci_alias_config_compute(self):
+        """Verify that the pci alias data is rendered properly.
+
+        Change pci-alias and assert that change propagates to the correct
+        file and that services are restarted as a result
+        """
+        # We are not touching the behavior of anything older than QUEENS
+        if self.current_release >= self.XENIAL_QUEENS:
+            self._test_pci_alias_config("nova-compute", ['nova-compute'])
+
     def test_500_hugepagereport_action(self):
         """Test hugepagereport action."""
         for unit in zaza.model.get_units('nova-compute',
@@ -152,22 +411,88 @@ class NovaCompute(test_utils.OpenStackBaseTest):
             self.assertFalse(int(run['Code']) == 0)
 
 
-class NovaCloudController(test_utils.OpenStackBaseTest):
+class NovaComputeActionTest(test_utils.OpenStackBaseTest):
+    """Run nova-compute specific tests.
+
+    Add this test class for new nova-compute action
+    to avoid breaking older version
+    """
+
+    def test_virsh_audit_action(self):
+        """Test virsh-audit action."""
+        for unit in zaza.model.get_units('nova-compute',
+                                         model_name=self.model_name):
+            logging.info('Running `virsh-audit` action'
+                         ' on  unit {}'.format(unit.entity_id))
+            action = zaza.model.run_action(
+                unit.entity_id,
+                'virsh-audit',
+                model_name=self.model_name,
+                action_params={})
+            if "failed" in action.data["status"]:
+                raise Exception(
+                    "The action failed: {}".format(action.data["message"]))
+
+
+class NovaCloudControllerActionTest(test_utils.OpenStackBaseTest):
+    """Run nova-cloud-controller specific tests.
+
+    Add this test class for new nova-cloud-controller action
+    to avoid breaking older version.
+    """
+
+    def test_sync_compute_az_action(self):
+        """Test sync-compute-availability-zones action."""
+        juju_units_az_map = {}
+        compute_config = zaza.model.get_application_config('nova-compute')
+        default_az = compute_config['default-availability-zone']['value']
+        use_juju_az = compute_config['customize-failure-domain']['value']
+
+        for unit in zaza.model.get_units('nova-compute',
+                                         model_name=self.model_name):
+            zone = default_az
+            if use_juju_az:
+                result = zaza.model.run_on_unit(unit.name,
+                                                'echo $JUJU_AVAILABILITY_ZONE',
+                                                model_name=self.model_name,
+                                                timeout=60)
+                self.assertEqual(int(result['Code']), 0)
+                juju_az = result['Stdout'].strip()
+                if juju_az:
+                    zone = juju_az
+
+            juju_units_az_map[unit.public_address] = zone
+            continue
+
+        session = openstack_utils.get_overcloud_keystone_session()
+        nova = openstack_utils.get_nova_session_client(session)
+
+        result = zaza.model.run_action_on_leader(
+            'nova-cloud-controller',
+            'sync-compute-availability-zones',
+            model_name=self.model_name)
+
+        # For validating the action results, we simply want to validate that
+        # the action was completed and we have something in the output. The
+        # functional validation really occurs below, in that the hosts are
+        # checked to be in the appropriate host aggregates.
+        self.assertEqual(result.status, 'completed')
+        self.assertNotEqual('', result.results['output'])
+
+        unique_az_list = list(set(juju_units_az_map.values()))
+        aggregates = nova.aggregates.list()
+        self.assertEqual(len(aggregates), len(unique_az_list))
+        for unit_address in juju_units_az_map:
+            az = juju_units_az_map[unit_address]
+            aggregate = nova.aggregates.find(
+                name='{}_az'.format(az), availability_zone=az)
+            hypervisor = nova.hypervisors.find(host_ip=unit_address)
+            self.assertIn(hypervisor.hypervisor_hostname, aggregate.hosts)
+
+
+class NovaCloudController(NovaCommonTests):
     """Run nova-cloud-controller specific tests."""
 
-    XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka')
-    XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata')
-    XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens')
-    BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens')
-    BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky')
-    BIONIC_TRAIN = openstack_utils.get_os_release('bionic_train')
-
-    @classmethod
-    def setUpClass(cls):
-        """Run class setup for running nova-cloud-controller tests."""
-        super(NovaCloudController, cls).setUpClass()
-        cls.current_release = openstack_utils.get_os_release()
-
     @property
     def services(self):
         """Return a list of services for the selected OpenStack release."""
@@ -263,70 +588,13 @@ class NovaCloudController(test_utils.OpenStackBaseTest):
                 'filter:legacy_ratelimit': {
                     'limits': ["( POST, '*', .*, 9999, MINUTE );"]}})
 
-    def test_310_pci_alias_config(self):
+    def test_310_pci_alias_config_ncc(self):
         """Verify that the pci alias data is rendered properly.
 
         Change pci-alias and assert that change propagates to the correct
         file and that services are restarted as a result
         """
-        logging.info('Checking pci aliases in nova config...')
-
-        # Expected default and alternate values
-        current_value = zaza.model.get_application_config(
-            'nova-cloud-controller')['pci-alias']
-        try:
-            current_value = current_value['value']
-        except KeyError:
-            current_value = None
-        new_value = '[{}, {}]'.format(
-            json.dumps({
-                'name': 'IntelNIC',
-                'capability_type': 'pci',
-                'product_id': '1111',
-                'vendor_id': '8086',
-                'device_type': 'type-PF'
-            }, sort_keys=True),
-            json.dumps({
-                'name': ' Cirrus Logic ',
-                'capability_type': 'pci',
-                'product_id': '0ff2',
-                'vendor_id': '10de',
-                'device_type': 'type-PCI'
-            }, sort_keys=True))
-
-        set_default = {'pci-alias': current_value}
-        set_alternate = {'pci-alias': new_value}
-
-        expected_conf_section = 'DEFAULT'
-        expected_conf_key = 'pci_alias'
-        if self.current_release >= self.XENIAL_OCATA:
-            expected_conf_section = 'pci'
-            expected_conf_key = 'alias'
-
-        default_entry = {expected_conf_section: {}}
-        alternate_entry = {expected_conf_section: {
-            expected_conf_key: [
-                ('{"capability_type": "pci", "device_type": "type-PF", '
-                 '"name": "IntelNIC", "product_id": "1111", '
-                 '"vendor_id": "8086"}'),
-                ('{"capability_type": "pci", "device_type": "type-PCI", '
-                 '"name": " Cirrus Logic ", "product_id": "0ff2", '
-                 '"vendor_id": "10de"}')]}}
-
-        # Config file affected by juju set config change
-        conf_file = '/etc/nova/nova.conf'
-
-        # Make config change, check for service restarts
-        logging.info(
-            'Setting config on nova-cloud-controller to {}'.format(
-                set_alternate))
-        self.restart_on_changed(
-            conf_file,
-            set_default,
-            set_alternate,
-            default_entry,
-            alternate_entry,
-            self.services)
+        self._test_pci_alias_config("nova-cloud-controller", self.services)
 
     def test_900_restart_on_config_change(self):
         """Checking restart happens on config change.
@@ -469,17 +737,24 @@ class SecurityTests(test_utils.OpenStackBaseTest):
         # Changes fixing the below expected failures will be made following
         # this initial work to get validation in. There will be bugs targeted
         # to each one and resolved independently where possible.
-
         expected_failures = [
-            'is-volume-encryption-enabled',
-            'validate-uses-tls-for-glance',
-            'validate-uses-tls-for-keystone',
         ]
         expected_passes = [
             'validate-file-ownership',
             'validate-file-permissions',
             'validate-uses-keystone',
         ]
+        tls_checks = [
+            'validate-uses-tls-for-glance',
+            'validate-uses-tls-for-keystone',
+        ]
+        if zaza.model.get_relation_id(
+                'nova-cloud-controller',
+                'vault',
+                remote_interface_name='certificates'):
+            expected_passes.extend(tls_checks)
+        else:
+            expected_failures.extend(tls_checks)
 
         for unit in zaza.model.get_units(self.application_name,
                                          model_name=self.model_name):
@@ -493,4 +768,4 @@ class SecurityTests(test_utils.OpenStackBaseTest):
                     action_params={}),
                 expected_passes,
                 expected_failures,
-                expected_to_pass=False)
+                expected_to_pass=not len(expected_failures))

zaza/openstack/charm_tests/octavia/setup.py

@@ -25,6 +25,9 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup
 import zaza.openstack.utilities.openstack as openstack
 import zaza.openstack.configure.guest
 
+import zaza.openstack.charm_tests.nova.setup as nova_setup
+import zaza.openstack.charm_tests.nova.utils as nova_utils
+
 
 def ensure_lts_images():
     """Ensure that bionic and focal images are available for the tests."""
@@ -51,13 +54,32 @@ def add_amphora_image(image_url=None):
 
 
 def configure_octavia():
-    """Do mandatory post deployment configuration of Octavia."""
-    # Tell Octavia charm it is safe to create cloud resources
-    logging.info('Running `configure-resources` action on Octavia leader unit')
-    zaza.model.run_action_on_leader(
-        'octavia',
-        'configure-resources',
-        action_params={})
+    """Do post deployment configuration and initialization of Octavia.
+
+    Certificates for the private Octavia worker <-> Amphorae communication must
+    be generated and set trough charm configuration.
+
+    The optional SSH configuration options are set to enable debug and log
+    collection from Amphorae, we will use the same keypair as Zaza uses for
+    instance creation.
+
+    The `configure-resources` action must be run to have the charm create
+    in-cloud resources such as management network and associated ports and
+    security groups.
+    """
+    # Set up Nova client to create/retrieve keypair for Amphora debug purposes.
+    #
+    # We reuse the Nova setup code for this and in most cases the test
+    # declaration will already defined that the Nova manage_ssh_key setup
+    # helper to run before we get here. Re-run here to make sure this setup
+    # function can be used separately, manage_ssh_key is idempotent.
+    keystone_session = openstack.get_overcloud_keystone_session()
+    nova_client = openstack.get_nova_session_client(
+        keystone_session)
+    nova_setup.manage_ssh_key(nova_client)
+    ssh_public_key = openstack.get_public_key(
+        nova_client, nova_utils.KEYPAIR_NAME)
+
     # Generate certificates for controller/load balancer instance communication
     (issuing_cakey, issuing_cacert) = cert.generate_cert(
         'OSCI Zaza Issuer',
@@ -71,7 +93,7 @@ def configure_octavia():
         issuer_name='OSCI Zaza Octavia Controller',
         signing_key=controller_cakey)
     controller_bundle = controller_cert + controller_key
-    cert_config = {
+    charm_config = {
         'lb-mgmt-issuing-cacert': base64.b64encode(
             issuing_cacert).decode('utf-8'),
         'lb-mgmt-issuing-ca-private-key': base64.b64encode(
@@ -81,22 +103,45 @@ def configure_octavia():
             controller_cacert).decode('utf-8'),
         'lb-mgmt-controller-cert': base64.b64encode(
             controller_bundle).decode('utf-8'),
+        'amp-ssh-key-name': 'octavia',
+        'amp-ssh-pub-key': base64.b64encode(
+            bytes(ssh_public_key, 'utf-8')).decode('utf-8'),
     }
-    logging.info('Configuring certificates for mandatory Octavia '
-                 'client/server authentication '
-                 '(client being the ``Amphorae`` load balancer instances)')
+
+    # Tell Octavia charm it is safe to create cloud resources, we do this now
+    # because the workload status will be checked on config-change and it gets
+    # a bit complicated to augment test config to accept 'blocked' vs. 'active'
+    # in the various stages.
+    logging.info('Running `configure-resources` action on Octavia leader unit')
+    zaza.model.run_action_on_leader(
+        'octavia',
+        'configure-resources',
+        action_params={})
 
     # Our expected workload status will change after we have configured the
     # certificates
     test_config = zaza.charm_lifecycle.utils.get_charm_config()
     del test_config['target_deploy_status']['octavia']
 
+    logging.info('Configuring certificates for mandatory Octavia '
+                 'client/server authentication '
+                 '(client being the ``Amphorae`` load balancer instances)')
+
     _singleton = zaza.openstack.charm_tests.test_utils.OpenStackBaseTest()
     _singleton.setUpClass(application_name='octavia')
-    with _singleton.config_change(cert_config, cert_config):
+    with _singleton.config_change(charm_config, charm_config):
         # wait for configuration to be applied then return
         pass
 
+    # Should we consider making the charm attempt to create this key on
+    # config-change?
+    logging.info('Running `configure-resources` action again to ensure '
+                 'Octavia Nova SSH key pair is created after config change.')
+    zaza.model.run_action_on_leader(
+        'octavia',
+        'configure-resources',
+        action_params={})
+
 
 def centralized_fip_network():
     """Create network with centralized router for connecting lb and fips.

zaza/openstack/charm_tests/octavia/tests.py

@@ -25,6 +25,8 @@ import osc_lib.exceptions
 import zaza.openstack.charm_tests.test_utils as test_utils
 import zaza.openstack.utilities.openstack as openstack_utils
 
+from zaza.openstack.utilities import ObjectRetrierWraps
+
 
 class CharmOperationTest(test_utils.OpenStackBaseTest):
     """Charm operation tests."""
@@ -63,12 +65,12 @@ class LBAASv2Test(test_utils.OpenStackBaseTest):
     def setUpClass(cls):
         """Run class setup for running LBaaSv2 service tests."""
         super(LBAASv2Test, cls).setUpClass()
-        cls.keystone_client = openstack_utils.get_keystone_session_client(
-            cls.keystone_session)
-        cls.neutron_client = openstack_utils.get_neutron_session_client(
-            cls.keystone_session)
-        cls.octavia_client = openstack_utils.get_octavia_session_client(
-            cls.keystone_session)
+        cls.keystone_client = ObjectRetrierWraps(
+            openstack_utils.get_keystone_session_client(cls.keystone_session))
+        cls.neutron_client = ObjectRetrierWraps(
+            openstack_utils.get_neutron_session_client(cls.keystone_session))
+        cls.octavia_client = ObjectRetrierWraps(
+            openstack_utils.get_octavia_session_client(cls.keystone_session))
         cls.RESOURCE_PREFIX = 'zaza-octavia'
 
         # NOTE(fnordahl): in the event of a test failure we do not want to run
@@ -280,7 +282,13 @@ class LBAASv2Test(test_utils.OpenStackBaseTest):
                 lambda x: octavia_client.member_show(
                     pool_id=pool_id, member_id=x),
                 member_id,
-                operating_status='ONLINE' if monitor else '')
+                operating_status='')
+            # Temporarily disable this check until we figure out why
+            # operational_status sometimes does not become 'ONLINE'
+            # while the member does indeed work and the subsequent
+            # retrieval of payload through loadbalancer is successful
+            # ref LP: #1896729.
+            #    operating_status='ONLINE' if monitor else '')
             logging.info(resp)
         return lb
 

zaza/openstack/charm_tests/openstack_dashboard/tests.py

@@ -67,7 +67,6 @@ def _login(dashboard_url, domain, username, password, cafile=None):
     # start session, get csrftoken
     client = requests.session()
     client.get(auth_url, verify=cafile)
-
     if 'csrftoken' in client.cookies:
         csrftoken = client.cookies['csrftoken']
     else:
@@ -163,7 +162,60 @@ def _do_request(request, cafile=None):
     return urllib.request.urlopen(request, cafile=cafile)
 
 
-class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
+class OpenStackDashboardBase():
+    """Mixin for interacting with Horizon."""
+
+    def get_base_url(self):
+        """Return the base url for http(s) requests.
+
+        :returns: URL
+        :rtype: str
+        """
+        vip = (zaza_model.get_application_config(self.application_name)
+               .get("vip").get("value"))
+        if vip:
+            ip = vip
+        else:
+            unit = zaza_model.get_unit_from_name(
+                zaza_model.get_lead_unit_name(self.application_name))
+            ip = unit.public_address
+
+        logging.debug("Dashboard ip is:{}".format(ip))
+        scheme = 'http'
+        if self.use_https:
+            scheme = 'https'
+        url = '{}://{}'.format(scheme, ip)
+        return url
+
+    def get_horizon_url(self):
+        """Return the url for acccessing horizon.
+
+        :returns: Horizon URL
+        :rtype: str
+        """
+        url = '{}/horizon'.format(self.get_base_url())
+        logging.info("Horizon URL is: {}".format(url))
+        return url
+
+    @property
+    def use_https(self):
+        """Whether dashboard is using https.
+
+        :returns: Whether dashboard is using https
+        :rtype: boolean
+        """
+        use_https = False
+        vault_relation = zaza_model.get_relation_id(
+            self.application,
+            'vault',
+            remote_interface_name='certificates')
+        if vault_relation:
+            use_https = True
+        return use_https
+
+
+class OpenStackDashboardTests(test_utils.OpenStackBaseTest,
+                              OpenStackDashboardBase):
     """Encapsulate openstack dashboard charm tests."""
 
     @classmethod
@@ -171,13 +223,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
         """Run class setup for running openstack dashboard charm tests."""
         super(OpenStackDashboardTests, cls).setUpClass()
         cls.application = 'openstack-dashboard'
-        cls.use_https = False
-        vault_relation = zaza_model.get_relation_id(
-            cls.application,
-            'vault',
-            remote_interface_name='certificates')
-        if vault_relation:
-            cls.use_https = True
 
     def test_050_local_settings_permissions_regression_check_lp1755027(self):
         """Assert regression check lp1755027.
@@ -302,39 +347,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
                         mismatches.append(msg)
         return mismatches
 
-    def get_base_url(self):
-        """Return the base url for http(s) requests.
-
-        :returns: URL
-        :rtype: str
-        """
-        vip = (zaza_model.get_application_config(self.application_name)
-               .get("vip").get("value"))
-        if vip:
-            ip = vip
-        else:
-            unit = zaza_model.get_unit_from_name(
-                zaza_model.get_lead_unit_name(self.application_name))
-            ip = unit.public_address
-
-        logging.debug("Dashboard ip is:{}".format(ip))
-        scheme = 'http'
-        if self.use_https:
-            scheme = 'https'
-        url = '{}://{}'.format(scheme, ip)
-        logging.debug("Base URL is: {}".format(url))
-        return url
-
-    def get_horizon_url(self):
-        """Return the url for acccessing horizon.
-
-        :returns: Horizon URL
-        :rtype: str
-        """
-        url = '{}/horizon'.format(self.get_base_url())
-        logging.info("Horizon URL is: {}".format(url))
-        return url
-
     def test_400_connection(self):
         """Test that dashboard responds to http request.
 
@@ -395,17 +407,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
                 self.assertEqual(e.code, 404, msg)
         logging.info('OK')
 
-    def test_501_security_checklist_action(self):
-        """Verify expected result on a default install.
-
-        Ported from amulet tests.
-        """
-        logging.info("Testing security-checklist")
-        unit_name = zaza_model.get_lead_unit_name('openstack-dashboard')
-        action = zaza_model.run_action(unit_name, 'security-checklist')
-        assert action.data.get(u"status") == "failed", \
-            "Security check is expected to not pass by default"
-
     def test_900_restart_on_config_change(self):
         """Verify that the specified services are restarted on config changed.
 
@@ -450,7 +451,8 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest):
             logging.info("Testing pause resume")
 
 
-class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization):
+class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization,
+                                     OpenStackDashboardBase):
     """Test the policyd override using the dashboard."""
 
     good = {
@@ -476,6 +478,7 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization):
         super(OpenStackDashboardPolicydTests, cls).setUpClass(
             application_name="openstack-dashboard")
         cls.application_name = "openstack-dashboard"
+        cls.application = cls.application_name
 
     def get_client_and_attempt_operation(self, ip):
         """Attempt to list users on the openstack-dashboard service.
@@ -500,9 +503,51 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization):
         username = 'admin',
         password = overcloud_auth['OS_PASSWORD'],
         client, response = _login(
-            unit.public_address, domain, username, password)
+            self.get_horizon_url(), domain, username, password)
         # now attempt to get the domains page
         _url = self.url.format(unit.public_address)
         result = client.get(_url)
         if result.status_code == 403:
             raise policyd.PolicydOperationFailedException("Not authenticated")
+
+
+class SecurityTests(test_utils.OpenStackBaseTest,
+                    OpenStackDashboardBase):
+    """Openstack-dashboard security tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running openstack-dashboard SecurityTests."""
+        super(SecurityTests, cls).setUpClass()
+
+    def test_security_checklist(self):
+        """Verify expected state with security checklist."""
+        logging.info("Testing security checklist.")
+
+        expected_failures = [
+            'csrf_cookie_set',
+            'disable_password_reveal',
+            'disallow-iframe-embed',
+            'password-validator-is-not-default',
+            'securie_proxy_ssl_header_is_set',
+            'session_cookie-httponly',
+            'session-cookie-store',
+        ]
+        expected_passes = [
+            'disable_password_autocomplete',
+            'enforce-password-check',
+            'validate-file-ownership',
+            'validate-file-permissions'
+        ]
+
+        logging.info('Running `security-checklist` action'
+                     ' on {} leader'.format(self.application_name))
+        test_utils.audit_assertions(
+            zaza_model.run_action_on_leader(
+                self.application_name,
+                'security-checklist',
+                model_name=self.model_name,
+                action_params={}),
+            expected_passes,
+            expected_failures,
+            expected_to_pass=False)

zaza/openstack/charm_tests/openstack_upgrade/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Canonical Ltd.
+# Copyright 2020 Canonical Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Test security checklist."""
+"""Code for testing openstack upgrades."""

zaza/openstack/charm_tests/openstack_upgrade/tests.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Define class for OpenStack Upgrade."""
+
+import logging
+import unittest
+
+from zaza.openstack.utilities import (
+    cli as cli_utils,
+    upgrade_utils as upgrade_utils,
+    openstack as openstack_utils,
+    openstack_upgrade as openstack_upgrade,
+)
+from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest
+
+
+class OpenStackUpgradeVMLaunchBase(object):
+    """A base class to peform a simple validation on the cloud.
+
+    This wraps an OpenStack upgrade with a VM launch before and after the
+    upgrade.
+
+    This test requires a full OpenStack including at least: keystone, glance,
+    nova-cloud-controller, nova-compute, neutron-gateway, neutron-api and
+    neutron-openvswitch.
+
+    This class should be used as a base class to the upgrade 'test'.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for OpenStack Upgrades."""
+        print("Running OpenStackUpgradeMixin setUpClass")
+        super().setUpClass()
+        cls.lts = LTSGuestCreateTest()
+        cls.lts.setUpClass()
+
+    def test_100_validate_pre_openstack_upgrade_cloud(self):
+        """Validate pre openstack upgrade."""
+        logging.info("Validate pre-openstack-upgrade: Spin up LTS instance")
+        self.lts.test_launch_small_instance()
+
+    def test_500_validate_openstack_upgraded_cloud(self):
+        """Validate post openstack upgrade."""
+        logging.info("Validate post-openstack-upgrade: Spin up LTS instance")
+        self.lts.test_launch_small_instance()
+
+
+class WaitForMySQL(unittest.TestCase):
+    """Helper test to wait on mysql-innodb-cluster to be fully ready.
+
+    In practice this means that there is at least on R/W unit available.
+    Sometimes, after restarting units in the mysql-innodb-cluster, all the
+    units are R/O until the cluster picks the R/W unit.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """Set up class."""
+        print("Running OpenstackUpgradeTests setUpClass")
+        super().setUpClass()
+        cli_utils.setup_logging()
+
+    def test_100_wait_for_happy_mysql_innodb_cluster(self):
+        """Wait for mysql cluster to have at least one R/W node."""
+        logging.info("Starting wait for an R/W unit.")
+        openstack_upgrade.block_until_mysql_innodb_cluster_has_rw()
+        logging.info("Done .. all seems well.")
+
+
+class OpenStackUpgradeTestsFocalUssuri(OpenStackUpgradeVMLaunchBase):
+    """Upgrade OpenStack from distro -> cloud:focal-victoria."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for OpenStack Upgrades."""
+        print("Running OpenstackUpgradeTests setUpClass")
+        super().setUpClass()
+        cli_utils.setup_logging()
+
+    def test_200_run_openstack_upgrade(self):
+        """Run openstack upgrade, but work out what to do."""
+        openstack_upgrade.run_upgrade_tests("cloud:focal-victoria")
+
+
+class OpenStackUpgradeTests(OpenStackUpgradeVMLaunchBase):
+    """A Principal Class to encapsulate OpenStack Upgrade Tests.
+
+    A generic Test class that can discover which Ubuntu version and OpenStack
+    version to upgrade from.
+
+    TODO: Not used at present.  Use the declarative tests directly that choose
+    the version to upgrade to.  The functions that this class depends on need a
+    bit more work regarding how the determine which version to go to.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for OpenStack Upgrades."""
+        print("Running OpenstackUpgradeTests setUpClass")
+        super().setUpClass()
+        cli_utils.setup_logging()
+
+    def test_200_run_openstack_upgrade(self):
+        """Run openstack upgrade, but work out what to do.
+
+        TODO: This is really inefficient at the moment, and doesn't (yet)
+        determine which ubuntu version to work from.  Don't use until we can
+        make it better.
+        """
+        # TODO: work out the most recent Ubuntu version; we assume this is the
+        # version that OpenStack is running on.
+        ubuntu_version = "focal"
+        logging.info("Getting all principle applications ...")
+        principle_services = upgrade_utils.get_all_principal_applications()
+        logging.info(
+            "Getting OpenStack vesions from principal applications ...")
+        current_versions = openstack_utils.get_current_os_versions(
+            principle_services)
+        logging.info("current versions: %s" % current_versions)
+        # Find the lowest value openstack release across all services and make
+        # sure all servcies are upgraded to one release higher than the lowest
+        from_version = upgrade_utils.get_lowest_openstack_version(
+            current_versions)
+        logging.info("from version: %s" % from_version)
+        to_version = upgrade_utils.determine_next_openstack_release(
+            from_version)[1]
+        logging.info("to version: %s" % to_version)
+        # TODO: need to determine the ubuntu base verion that is being upgraded
+        target_source = upgrade_utils.determine_new_source(
+            ubuntu_version, from_version, to_version, single_increment=True)
+        logging.info("target source: %s" % target_source)
+        assert target_source is not None
+        openstack_upgrade.run_upgrade_tests(target_source)

zaza/openstack/charm_tests/ovn/tests.py

@@ -15,11 +15,14 @@
 """Encapsulate OVN testing."""
 
 import logging
-import tenacity
 
 import juju
 
+import tenacity
+
 import zaza
+
+import zaza.model
 import zaza.openstack.charm_tests.test_utils as test_utils
 import zaza.openstack.utilities.generic as generic_utils
 import zaza.openstack.utilities.openstack as openstack_utils
@@ -36,10 +39,19 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest):
         """Run class setup for OVN charm operation tests."""
         super(BaseCharmOperationTest, cls).setUpClass()
         cls.services = ['NotImplemented']  # This must be overridden
+        cls.nrpe_checks = ['NotImplemented']  # This must be overridden
         cls.current_release = openstack_utils.get_os_release(
             openstack_utils.get_current_os_release_pair(
                 cls.release_application or cls.application_name))
 
+    @tenacity.retry(
+        retry=tenacity.retry_if_result(lambda ret: ret is not None),
+        # sleep for 2mins to allow 1min cron job to run...
+        wait=tenacity.wait_fixed(120),
+        stop=tenacity.stop_after_attempt(2))
+    def _retry_check_commands_on_units(self, cmds, units):
+        return generic_utils.check_commands_on_units(cmds, units)
+
     def test_pause_resume(self):
         """Run pause and resume tests.
 
@@ -50,6 +62,20 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest):
             logging.info('Testing pause resume (services="{}")'
                          .format(self.services))
 
+    def test_nrpe_configured(self):
+        """Confirm that the NRPE service check files are created."""
+        units = zaza.model.get_units(self.application_name)
+        cmds = []
+        for check_name in self.nrpe_checks:
+            cmds.append(
+                'egrep -oh /usr/local.* /etc/nagios/nrpe.d/'
+                'check_{}.cfg'.format(check_name)
+            )
+        ret = self._retry_check_commands_on_units(cmds, units)
+        if ret:
+            logging.info(ret)
+        self.assertIsNone(ret, msg=ret)
+
 
 class CentralCharmOperationTest(BaseCharmOperationTest):
     """OVN Central Charm operation tests."""
@@ -62,6 +88,22 @@ class CentralCharmOperationTest(BaseCharmOperationTest):
             'ovn-northd',
             'ovsdb-server',
         ]
+        source = zaza.model.get_application_config(
+            cls.application_name)['source']['value']
+        logging.info(source)
+        if 'train' in source:
+            cls.nrpe_checks = [
+                'ovn-northd',
+                'ovn-nb-ovsdb',
+                'ovn-sb-ovsdb',
+            ]
+        else:
+            # Ussuri or later (distro or cloudarchive)
+            cls.nrpe_checks = [
+                'ovn-northd',
+                'ovn-ovsdb-server-sb',
+                'ovn-ovsdb-server-nb',
+            ]
 
 
 class ChassisCharmOperationTest(BaseCharmOperationTest):
@@ -76,11 +118,38 @@ class ChassisCharmOperationTest(BaseCharmOperationTest):
         cls.services = [
             'ovn-controller',
         ]
+        if cls.application_name == 'ovn-chassis':
+            principal_app_name = 'magpie'
+        else:
+            principal_app_name = cls.application_name
+        source = zaza.model.get_application_config(
+            principal_app_name)['source']['value']
+        logging.info(source)
+        if 'train' in source:
+            cls.nrpe_checks = [
+                'ovn-host',
+                'ovs-vswitchd',
+                'ovsdb-server',
+            ]
+        else:
+            # Ussuri or later (distro or cloudarchive)
+            cls.nrpe_checks = [
+                'ovn-controller',
+                'ovsdb-server',
+                'ovs-vswitchd',
+            ]
 
 
 class OVSOVNMigrationTest(test_utils.BaseCharmTest):
     """OVS to OVN migration tests."""
 
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for OVN migration tests."""
+        super(OVSOVNMigrationTest, cls).setUpClass()
+        cls.current_release = openstack_utils.get_os_release(
+            openstack_utils.get_current_os_release_pair())
+
     def setUp(self):
         """Perform migration steps prior to validation."""
         super(OVSOVNMigrationTest, self).setUp()
@@ -348,3 +417,93 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest):
         zaza.model.wait_for_agent_status()
         zaza.model.wait_for_application_states(
             states=self.target_deploy_status)
+        # Workaround for our old friend LP: #1852221 which hit us again on
+        # Groovy. We make the os_release check explicit so that we can
+        # re-evaluate the need for the workaround at the next release.
+        if self.current_release == openstack_utils.get_os_release(
+                'groovy_victoria'):
+            try:
+                for application in ('ovn-chassis', 'ovn-dedicated-chassis'):
+                    for unit in zaza.model.get_units(application):
+                        zaza.model.run_on_unit(
+                            unit.entity_id,
+                            'systemctl restart ovs-vswitchd')
+            except KeyError:
+                # One of the applications is not in the model, which is fine
+                pass
+
+
+class OVNChassisDeferredRestartTest(test_utils.BaseDeferredRestartTest):
+    """Deferred restart tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for deferred restart tests."""
+        super().setUpClass(application_name='ovn-chassis')
+
+    def run_tests(self):
+        """Run deferred restart tests."""
+        # Trigger a config change which triggers a deferred hook.
+        self.run_charm_change_hook_test('configure_ovs')
+
+        # Trigger a package change which requires a restart
+        self.run_package_change_test(
+            'openvswitch-switch',
+            'openvswitch-switch')
+
+    def get_new_config(self):
+        """Return the config key and new value to trigger a hook execution.
+
+        :returns: Config key and new value
+        :rtype: (str, bool)
+        """
+        app_config = zaza.model.get_application_config(self.application_name)
+        return 'enable-sriov', str(not app_config['enable-sriov']['value'])
+
+
+class OVNDedicatedChassisDeferredRestartTest(
+        test_utils.BaseDeferredRestartTest):
+    """Deferred restart tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for deferred restart tests."""
+        super().setUpClass(application_name='ovn-dedicated-chassis')
+
+    def run_tests(self):
+        """Run deferred restart tests."""
+        # Trigger a config change which triggers a deferred hook.
+        self.run_charm_change_hook_test('configure_ovs')
+
+        # Trigger a package change which requires a restart
+        self.run_package_change_test(
+            'openvswitch-switch',
+            'openvswitch-switch')
+
+    def get_new_config(self):
+        """Return the config key and new value to trigger a hook execution.
+
+        :returns: Config key and new value
+        :rtype: (str, bool)
+        """
+        app_config = zaza.model.get_application_config(self.application_name)
+        new_value = str(not app_config['disable-mlockall'].get('value', False))
+        return 'disable-mlockall', new_value
+
+
+class OVNCentralDeferredRestartTest(
+        test_utils.BaseDeferredRestartTest):
+    """Deferred restart tests for OVN Central."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for deferred restart tests."""
+        super().setUpClass(application_name='ovn-central')
+
+    def run_tests(self):
+        """Run deferred restart tests."""
+        # Charm does not defer hooks so that test is not included.
+        # Trigger a package change which requires a restart
+        self.run_package_change_test(
+            'ovn-central',
+            'ovn-central')

zaza/openstack/charm_tests/policyd/tests.py

@@ -337,8 +337,7 @@ class BasePolicydSpecialization(PolicydTest,
         logging.info('Authentication for {} on keystone IP {}'
                      .format(openrc['OS_USERNAME'], ip))
         if self.tls_rid:
-            openrc['OS_CACERT'] = \
-                openstack_utils.KEYSTONE_LOCAL_CACERT
+            openrc['OS_CACERT'] = openstack_utils.get_cacert()
             openrc['OS_AUTH_URL'] = (
                 openrc['OS_AUTH_URL'].replace('http', 'https'))
         logging.info('keystone IP {}'.format(ip))
@@ -402,9 +401,10 @@ class BasePolicydSpecialization(PolicydTest,
     def test_003_test_override_is_observed(self):
         """Test that the override is observed by the underlying service."""
         if (openstack_utils.get_os_release() <
-                openstack_utils.get_os_release('groovy_victoria')):
+                openstack_utils.get_os_release('xenial_queens')):
             raise unittest.SkipTest(
-                "Test skipped until Bug #1880959 is fix released")
+                "Test skipped because bug #1880959 won't be fixed for "
+                "releases older than Queens")
         if self._test_name is None:
             logging.info("Doing policyd override for {}"
                          .format(self._service_name))
@@ -572,6 +572,13 @@ class GlanceTests(BasePolicydSpecialization):
         super(GlanceTests, cls).setUpClass(application_name="glance")
         cls.application_name = "glance"
 
+    # NOTE(lourot): Same as NeutronApiTests. There is a race between the glance
+    # charm signalling its readiness and the service actually being ready to
+    # serve requests. The test will fail intermittently unless we gracefully
+    # accept this.
+    # Issue: openstack-charmers/zaza-openstack-tests#578
+    @tenacity.retry(wait=tenacity.wait_fixed(1),
+                    reraise=True, stop=tenacity.stop_after_delay(8))
     def get_client_and_attempt_operation(self, ip):
         """Attempt to list the images as a policyd override.
 
@@ -681,5 +688,6 @@ class OctaviaTests(BasePolicydSpecialization):
             self.get_keystone_session_admin_user(ip))
         try:
             octavia_client.provider_list()
-        except octaviaclient.OctaviaClientException:
+        except (octaviaclient.OctaviaClientException,
+                keystoneauth1.exceptions.http.Forbidden):
             raise PolicydOperationFailedException()

zaza/openstack/charm_tests/rabbitmq_server/tests.py

@@ -428,3 +428,58 @@ class RmqTests(test_utils.OpenStackBaseTest):
         check_units(all_units)
 
         logging.info('OK')
+
+
+class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest):
+    """Deferred restart tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for deferred restart tests."""
+        super().setUpClass(application_name='rabbitmq-server')
+
+    def check_status_message_is_clear(self):
+        """Check each units status message show no defeerred events."""
+        pattern = '(Unit is ready|Unit is ready and clustered)$'
+        for unit in zaza.model.get_units(self.application_name):
+            zaza.model.block_until_unit_wl_message_match(
+                unit.entity_id,
+                pattern)
+        zaza.model.block_until_all_units_idle()
+
+    def get_new_config(self):
+        """Return the config key and new value to trigger a hook execution.
+
+        :returns: Config key and new value
+        :rtype: (str, bool)
+        """
+        app_config = zaza.model.get_application_config(self.application_name)
+        new_value = str(int(
+            app_config['connection-backlog'].get('value', 100) + 1))
+        return 'connection-backlog', new_value
+
+    def run_tests(self):
+        """Run deferred restart tests."""
+        # Trigger a config change which triggers a deferred hook.
+        self.run_charm_change_hook_test('config-changed')
+
+        # Trigger a package change which requires a restart
+        self.run_package_change_test(
+            'rabbitmq-server',
+            'rabbitmq-server')
+
+    def check_clear_restarts(self):
+        """Clear and deferred restarts and check status.
+
+        Clear and deferred restarts and then check the workload status message
+        for each unit.
+        """
+        # Use action to run any deferred restarts
+        for unit in zaza.model.get_units(self.application_name):
+            zaza.model.run_action(
+                unit.entity_id,
+                'restart-services',
+                action_params={'services': 'rabbitmq-server'})
+
+        # Check workload status no longer shows deferred restarts.
+        self.check_status_message_is_clear()

zaza/openstack/charm_tests/saml_mellon/setup.py

@@ -25,6 +25,7 @@ from zaza.openstack.utilities import (
     cert as cert_utils,
     cli as cli_utils,
     openstack as openstack_utils,
+    generic as generic_utils,
 )
 
 
@@ -34,8 +35,8 @@ FEDERATED_DOMAIN = "federated_domain"
 FEDERATED_GROUP = "federated_users"
 MEMBER = "Member"
 IDP = "samltest"
+LOCAL_IDP_REMOTE_ID = "http://{}/simplesaml/saml2/idp/metadata.php"
 REMOTE_ID = "https://samltest.id/saml/idp"
-MAP_NAME = "{}_mapping".format(IDP)
 PROTOCOL_NAME = "mapped"
 MAP_TEMPLATE = '''
     [{{
@@ -45,7 +46,7 @@ MAP_TEMPLATE = '''
                         "name": "{{0}}"
                     }},
                     "group": {{
-                        "name": "federated_users",
+                        "name": "{group_id}",
                         "domain": {{
                             "id": "{domain_id}"
                         }}
@@ -55,7 +56,7 @@ MAP_TEMPLATE = '''
                         "name": "{{0}}_project",
                         "roles": [
                                      {{
-                                         "name": "Member"
+                                         "name": "{role_name}"
                                      }}
                                  ]
                     }}
@@ -81,7 +82,10 @@ SP_SIGNING_KEY_INFO_XML_TEMPLATE = '''
 '''
 
 
-def keystone_federation_setup():
+def keystone_federation_setup(federated_domain=FEDERATED_DOMAIN,
+                              federated_group=FEDERATED_GROUP,
+                              idp_name=IDP,
+                              idp_remote_id=REMOTE_ID):
     """Configure Keystone Federation."""
     cli_utils.setup_logging()
     keystone_session = openstack_utils.get_overcloud_keystone_session()
@@ -89,19 +93,19 @@ def keystone_federation_setup():
         keystone_session)
 
     try:
-        domain = keystone_client.domains.find(name=FEDERATED_DOMAIN)
+        domain = keystone_client.domains.find(name=federated_domain)
     except keystoneauth1.exceptions.http.NotFound:
         domain = keystone_client.domains.create(
-            FEDERATED_DOMAIN,
+            federated_domain,
             description="Federated Domain",
             enabled=True)
 
     try:
         group = keystone_client.groups.find(
-            name=FEDERATED_GROUP, domain=domain)
+            name=federated_group, domain=domain)
     except keystoneauth1.exceptions.http.NotFound:
         group = keystone_client.groups.create(
-            FEDERATED_GROUP,
+            federated_group,
             domain=domain,
             enabled=True)
 
@@ -109,30 +113,33 @@ def keystone_federation_setup():
     keystone_client.roles.grant(role, group=group, domain=domain)
 
     try:
-        idp = keystone_client.federation.identity_providers.find(
-            name=IDP, domain_id=domain.id)
+        idp = keystone_client.federation.identity_providers.get(idp_name)
     except keystoneauth1.exceptions.http.NotFound:
         idp = keystone_client.federation.identity_providers.create(
-            IDP,
-            remote_ids=[REMOTE_ID],
+            idp_name,
+            remote_ids=[idp_remote_id],
             domain_id=domain.id,
             enabled=True)
 
-    JSON_RULES = json.loads(MAP_TEMPLATE.format(domain_id=domain.id))
+    JSON_RULES = json.loads(MAP_TEMPLATE.format(
+        domain_id=domain.id, group_id=group.id, role_name=MEMBER))
 
+    map_name = "{}_mapping".format(idp_name)
     try:
-        keystone_client.federation.mappings.find(name=MAP_NAME)
+        keystone_client.federation.mappings.get(map_name)
     except keystoneauth1.exceptions.http.NotFound:
         keystone_client.federation.mappings.create(
-            MAP_NAME, rules=JSON_RULES)
+            map_name, rules=JSON_RULES)
 
     try:
-        keystone_client.federation.protocols.get(IDP, PROTOCOL_NAME)
+        keystone_client.federation.protocols.get(idp_name, PROTOCOL_NAME)
     except keystoneauth1.exceptions.http.NotFound:
         keystone_client.federation.protocols.create(
-            PROTOCOL_NAME, mapping=MAP_NAME, identity_provider=idp)
+            PROTOCOL_NAME, mapping=map_name, identity_provider=idp)
 
 
+# This setup method is deprecated. It will be removed once we fully drop the
+# `samltest.id` dependency.
 def attach_saml_resources(application="keystone-saml-mellon"):
     """Attach resource to the Keystone SAML Mellon charm."""
     test_idp_metadata_xml = "samltest.xml"
@@ -161,3 +168,83 @@ def attach_saml_resources(application="keystone-saml-mellon"):
         fp.flush()
         zaza.model.attach_resource(
             application, sp_signing_keyinfo_name, fp.name)
+
+
+def _attach_saml_resources_local_idp(keystone_saml_mellon_app_name=None,
+                                     test_saml_idp_app_name=None):
+    """Attach resources to the Keystone SAML Mellon and the local IdP."""
+    action_result = zaza.model.run_action_on_leader(
+        test_saml_idp_app_name, 'get-idp-metadata')
+    idp_metadata = action_result.data['results']['output']
+
+    generic_utils.attach_file_resource(
+        keystone_saml_mellon_app_name,
+        'idp-metadata',
+        idp_metadata,
+        '.xml')
+
+    (key, cert) = cert_utils.generate_cert('SP Signing Key')
+
+    cert = cert.decode().strip("-----BEGIN CERTIFICATE-----")
+    cert = cert.strip("-----END CERTIFICATE-----")
+
+    generic_utils.attach_file_resource(
+        keystone_saml_mellon_app_name,
+        'sp-private-key',
+        key.decode(),
+        '.pem')
+    generic_utils.attach_file_resource(
+        keystone_saml_mellon_app_name,
+        'sp-signing-keyinfo',
+        SP_SIGNING_KEY_INFO_XML_TEMPLATE.format(cert),
+        '.xml')
+
+    action_result = zaza.model.run_action_on_leader(
+        keystone_saml_mellon_app_name, 'get-sp-metadata')
+    sp_metadata = action_result.data['results']['output']
+
+    generic_utils.attach_file_resource(
+        test_saml_idp_app_name,
+        'sp-metadata',
+        sp_metadata,
+        '.xml')
+
+
+def attach_saml_resources_idp1():
+    """Attach the SAML resources for the local IdP #1."""
+    _attach_saml_resources_local_idp(
+        keystone_saml_mellon_app_name="keystone-saml-mellon1",
+        test_saml_idp_app_name="test-saml-idp1")
+
+
+def attach_saml_resources_idp2():
+    """Attach the SAML resources for the local IdP #2."""
+    _attach_saml_resources_local_idp(
+        keystone_saml_mellon_app_name="keystone-saml-mellon2",
+        test_saml_idp_app_name="test-saml-idp2")
+
+
+def keystone_federation_setup_idp1():
+    """Configure Keystone Federation for the local IdP #1."""
+    test_saml_idp_unit = zaza.model.get_units("test-saml-idp1")[0]
+    idp_remote_id = LOCAL_IDP_REMOTE_ID.format(
+        test_saml_idp_unit.public_address)
+
+    keystone_federation_setup(
+        federated_domain="federated_domain_idp1",
+        federated_group="federated_users_idp1",
+        idp_name="test-saml-idp1",
+        idp_remote_id=idp_remote_id)
+
+
+def keystone_federation_setup_idp2():
+    """Configure Keystone Federation for the local IdP #2."""
+    test_saml_idp_unit = zaza.model.get_units("test-saml-idp2")[0]
+    idp_remote_id = LOCAL_IDP_REMOTE_ID.format(
+        test_saml_idp_unit.public_address)
+
+    keystone_federation_setup(
+        federated_domain="federated_domain_idp2",
+        federated_group="federated_users_idp2",
+        idp_name="test-saml-idp2",
+        idp_remote_id=idp_remote_id)

zaza/openstack/charm_tests/saml_mellon/tests.py

@@ -30,6 +30,8 @@ class FailedToReachIDP(Exception):
     pass
 
 
+# This testing class is deprecated. It will be removed once we fully drop the
+# `samltest.id` dependency.
 class CharmKeystoneSAMLMellonTest(BaseKeystoneTest):
     """Charm Keystone SAML Mellon tests."""
 
@@ -156,3 +158,200 @@ class CharmKeystoneSAMLMellonTest(BaseKeystoneTest):
         # We may need to try/except to allow horizon to build its pages
         _do_redirect_check(url, region, idp_expect, horizon_expect)
         logging.info("SUCCESS")
+
+
+class BaseCharmKeystoneSAMLMellonTest(BaseKeystoneTest):
+    """Charm Keystone SAML Mellon tests."""
+
+    @classmethod
+    def setUpClass(cls,
+                   application_name="keystone-saml-mellon",
+                   test_saml_idp_app_name="test-saml-idp",
+                   horizon_idp_option_name="myidp_mapped",
+                   horizon_idp_display_name="myidp via mapped"):
+        """Run class setup for running Keystone SAML Mellon charm tests."""
+        super(BaseCharmKeystoneSAMLMellonTest, cls).setUpClass()
+        # Note: The BaseKeystoneTest class sets the application_name to
+        # "keystone" which breaks keystone-saml-mellon actions. Explicitly set
+        # application name here.
+        cls.application_name = application_name
+        cls.test_saml_idp_app_name = test_saml_idp_app_name
+        cls.horizon_idp_option_name = horizon_idp_option_name
+        cls.horizon_idp_display_name = horizon_idp_display_name
+        cls.action = "get-sp-metadata"
+        cls.current_release = openstack_utils.get_os_release()
+        cls.FOCAL_USSURI = openstack_utils.get_os_release("focal_ussuri")
+
+    @staticmethod
+    def check_horizon_redirect(horizon_url, horizon_expect,
+                               horizon_idp_option_name, horizon_region,
+                               idp_url, idp_expect):
+        """Validate the Horizon -> Keystone -> IDP redirects.
+
+        This validation is done through `requests.session()`, and the proper
+        get / post http calls.
+
+        :param horizon_url: The login page for the Horizon OpenStack dashboard.
+        :type horizon_url: string
+        :param horizon_expect: Information that needs to be displayed by
+                               Horizon login page, when there is a proper
+                               SAML IdP configuration.
+        :type horizon_expect: string
+        :param horizon_idp_option_name: The name of the IdP that is chosen
+                                        in the Horizon dropdown from the login
+                                        screen. This will go in the post body
+                                        as 'auth_type'.
+        :type horizon_idp_option_name: string
+        :param horizon_region: Information needed to complete the http post
+                               data for the Horizon login.
+        :type horizon_region: string
+        :param idp_url: The url for the IdP where the user needs to be
+                        redirected.
+        :type idp_url: string
+        :param idp_expect: Information that needs to be displayed by the IdP
+                           after the user is redirected there.
+        :type idp_expect: string
+        :returns: None
+        """
+        # start session, get csrftoken
+        client = requests.session()
+        # Verify=False see note below
+        login_page = client.get(horizon_url, verify=False)
+
+        # Validate SAML method is available
+        assert horizon_expect in login_page.text
+
+        # Get cookie
+        if "csrftoken" in client.cookies:
+            csrftoken = client.cookies["csrftoken"]
+        else:
+            raise Exception("Missing csrftoken")
+
+        # Build and send post request
+        form_data = {
+            "auth_type": horizon_idp_option_name,
+            "csrfmiddlewaretoken": csrftoken,
+            "next": "/horizon/project/api_access",
+            "region": horizon_region,
+        }
+
+        # Verify=False due to CA certificate bundles.
+        # If we don't set it validation fails for keystone/horizon
+        # We would have to install the keystone CA onto the system
+        # to validate end to end.
+        response = client.post(
+            horizon_url,
+            data=form_data,
+            headers={"Referer": horizon_url},
+            allow_redirects=True,
+            verify=False)
+
+        if idp_expect not in response.text:
+            msg = "FAILURE code={} text={}".format(response, response.text)
+            # Raise a custom exception.
+            raise FailedToReachIDP(msg)
+
+        # Validate that we were redirected to the proper IdP
+        assert response.url.startswith(idp_url)
+        assert idp_url in response.text
+
+    def test_run_get_sp_metadata_action(self):
+        """Validate the get-sp-metadata action."""
+        unit = zaza.model.get_units(self.application_name)[0]
+        ip = self.vip if self.vip else unit.public_address
+
+        action = zaza.model.run_action(unit.entity_id, self.action)
+        self.assertNotIn(
+            "failed",
+            action.data["status"],
+            msg="The action failed: {}".format(action.data["message"]))
+
+        output = action.data["results"]["output"]
+        root = etree.fromstring(output)
+        for item in root.items():
+            if "entityID" in item[0]:
+                self.assertIn(ip, item[1])
+
+        for appt in root.getchildren():
+            for elem in appt.getchildren():
+                for item in elem.items():
+                    if "Location" in item[0]:
+                        self.assertIn(ip, item[1])
+
+        logging.info("Successul get-sp-metadata action")
+
+    def test_saml_mellon_redirects(self):
+        """Validate the horizon -> keystone -> IDP redirects."""
+        unit = zaza.model.get_units(self.application_name)[0]
+        keystone_ip = self.vip if self.vip else unit.public_address
+
+        horizon = "openstack-dashboard"
+        horizon_config = zaza.model.get_application_config(horizon)
+        horizon_vip = horizon_config.get("vip").get("value")
+        unit = zaza.model.get_units("openstack-dashboard")[0]
+
+        horizon_ip = horizon_vip if horizon_vip else unit.public_address
+        proto = "https" if self.tls_rid else "http"
+
+        # Use Keystone URL for < Focal
+        if self.current_release < self.FOCAL_USSURI:
+            region = "{}://{}:5000/v3".format(proto, keystone_ip)
+        else:
+            region = "default"
+
+        idp_address = zaza.model.get_units(
+            self.test_saml_idp_app_name)[0].public_address
+
+        horizon_url = "{}://{}/horizon/auth/login/".format(proto, horizon_ip)
+        horizon_expect = '<option value="{0}">{1}</option>'.format(
+            self.horizon_idp_option_name, self.horizon_idp_display_name)
+        idp_url = ("http://{0}/simplesaml/"
+                   "module.php/core/loginuserpass.php").format(idp_address)
+        # This is the message the local test-saml-idp displays after you are
+        # redirected. It shows we have been directed to:
+        # horizon -> keystone -> test-saml-idp
+        idp_expect = (
+            "A service has requested you to authenticate yourself. Please "
+            "enter your username and password in the form below.")
+
+        # Execute the check
+        BaseCharmKeystoneSAMLMellonTest.check_horizon_redirect(
+            horizon_url=horizon_url,
+            horizon_expect=horizon_expect,
+            horizon_idp_option_name=self.horizon_idp_option_name,
+            horizon_region=region,
+            idp_url=idp_url,
+            idp_expect=idp_expect)
+        logging.info("SUCCESS")
+
+
+class CharmKeystoneSAMLMellonIDP1Test(BaseCharmKeystoneSAMLMellonTest):
+    """Charm Keystone SAML Mellon tests class for the local IDP #1."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running Keystone SAML Mellon charm tests.
+
+        It does the necessary setup for the local IDP #1.
+        """
+        super(CharmKeystoneSAMLMellonIDP1Test, cls).setUpClass(
+            application_name="keystone-saml-mellon1",
+            test_saml_idp_app_name="test-saml-idp1",
+            horizon_idp_option_name="test-saml-idp1_mapped",
+            horizon_idp_display_name="Test SAML IDP #1")
+
+
+class CharmKeystoneSAMLMellonIDP2Test(BaseCharmKeystoneSAMLMellonTest):
+    """Charm Keystone SAML Mellon tests class for the local IDP #2."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running Keystone SAML Mellon charm tests.
+
+        It does the necessary setup for the local IDP #2.
+        """
+        super(CharmKeystoneSAMLMellonIDP2Test, cls).setUpClass(
+            application_name="keystone-saml-mellon2",
+            test_saml_idp_app_name="test-saml-idp2",
+            horizon_idp_option_name="test-saml-idp2_mapped",
+            horizon_idp_display_name="Test SAML IDP #2")

zaza/openstack/charm_tests/security/tests.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2018 Canonical Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Encapsulate general security testing."""
-
-import unittest
-
-import zaza.model as model
-import zaza.charm_lifecycle.utils as utils
-from zaza.openstack.utilities.file_assertions import (
-    assert_path_glob,
-    assert_single_file,
-)
-
-
-def _make_test_function(application, file_details, paths=None):
-    """Generate a test function given the specified inputs.
-
-    :param application: Application name to assert file ownership on
-    :type application: str
-    :param file_details: Dictionary of file details to test
-    :type file_details: dict
-    :param paths: List of paths to test in this application
-    :type paths: Optional[list(str)]
-    :returns: Test function
-    :rtype: unittest.TestCase
-    """
-    def test(self):
-        for unit in model.get_units(application):
-            unit = unit.entity_id
-            if '*' in file_details['path']:
-                assert_path_glob(self, unit, file_details, paths)
-            else:
-                assert_single_file(self, unit, file_details)
-    return test
-
-
-def _add_tests():
-    """Add tests to the unittest.TestCase."""
-    def class_decorator(cls):
-        """Add tests based on input yaml to `cls`."""
-        files = utils.get_charm_config('./file-assertions.yaml')
-        deployed_applications = model.sync_deployed()
-        for name, attributes in files.items():
-            # Lets make sure to only add tests for deployed applications
-            if name in deployed_applications:
-                paths = [
-                    file['path'] for
-                    file in attributes['files']
-                    if "*" not in file["path"]
-                ]
-                for file in attributes['files']:
-                    test_func = _make_test_function(name, file, paths=paths)
-                    setattr(
-                        cls,
-                        'test_{}_{}'.format(name, file['path']),
-                        test_func)
-        return cls
-    return class_decorator
-
-
-class FileOwnershipTest(unittest.TestCase):
-    """Encapsulate File ownership tests."""
-
-    pass
-
-
-FileOwnershipTest = _add_tests()(FileOwnershipTest)

zaza/openstack/charm_tests/series_upgrade/parallel_tests.py

@@ -21,6 +21,7 @@ import logging
 import os
 import sys
 import unittest
+import juju
 
 from zaza import model
 from zaza.openstack.utilities import (
@@ -55,6 +56,12 @@ class ParallelSeriesUpgradeTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         """Run setup for Series Upgrades."""
+        # NOTE(ajkavanagh): Set the jujulib Connection frame size to 4GB to
+        # cope with all the outputs from series upgrade; long term, don't send
+        # that output back, which will require that the upgrade function in the
+        # charm doesn't capture the output of the upgrade in the action, but
+        # instead puts it somewhere that can by "juju scp"ed.
+        juju.client.connection.Connection.MAX_FRAME_SIZE = 2**32
         cli_utils.setup_logging()
         cls.from_series = None
         cls.to_series = None
@@ -89,19 +96,25 @@ class ParallelSeriesUpgradeTest(unittest.TestCase):
                 upgrade_function = \
                     parallel_series_upgrade.parallel_series_upgrade
 
+            # allow up to 4 parallel upgrades at a time.  This is to limit the
+            # amount of data/calls that asyncio is handling as it's gets
+            # unstable if all the applications are done at the same time.
+            sem = asyncio.Semaphore(4)
             for charm_name in apps:
                 charm = applications[charm_name]['charm']
                 name = upgrade_utils.extract_charm_name_from_url(charm)
                 upgrade_config = parallel_series_upgrade.app_config(name)
                 upgrade_functions.append(
-                    upgrade_function(
-                        charm_name,
-                        **upgrade_config,
-                        from_series=from_series,
-                        to_series=to_series,
-                        completed_machines=completed_machines,
-                        workaround_script=workaround_script,
-                        files=files))
+                    wrap_coroutine_with_sem(
+                        sem,
+                        upgrade_function(
+                            charm_name,
+                            **upgrade_config,
+                            from_series=from_series,
+                            to_series=to_series,
+                            completed_machines=completed_machines,
+                            workaround_script=workaround_script,
+                            files=files)))
             asyncio.get_event_loop().run_until_complete(
                 asyncio.gather(*upgrade_functions))
             model.block_until_all_units_idle()
@@ -109,6 +122,18 @@ class ParallelSeriesUpgradeTest(unittest.TestCase):
         logging.info("Done!")
 
 
+async def wrap_coroutine_with_sem(sem, coroutine):
+    """Wrap a coroutine with a semaphore to limit concurrency.
+
+    :param sem: The semaphore to limit concurrency
+    :type sem: asyncio.Semaphore
+    :param coroutine: the corouting to limit concurrency
+    :type coroutine: types.CoroutineType
+    """
+    async with sem:
+        await coroutine
+
+
 class OpenStackParallelSeriesUpgrade(ParallelSeriesUpgradeTest):
     """OpenStack Series Upgrade.
 

zaza/openstack/charm_tests/tempest/setup.py

@@ -17,6 +17,7 @@
 import jinja2
 import urllib.parse
 import os
+import subprocess
 
 import zaza.utilities.deployment_env as deployment_env
 import zaza.openstack.utilities.juju as juju_utils
@@ -170,6 +171,27 @@ def add_keystone_config(ctxt, keystone_session):
     ctxt['default_domain_id'] = domain.id
 
 
+def add_octavia_config(ctxt):
+    """Add octavia config to context.
+
+    :param ctxt: Context dictionary
+    :type ctxt: dict
+    :returns: None
+    :rtype: None
+    :raises: subprocess.CalledProcessError
+    """
+    subprocess.check_call([
+        'curl',
+        "{}:80/swift/v1/fixtures/test_server.bin".format(
+            ctxt['test_swift_ip']),
+        '-o', "{}/test_server.bin".format(ctxt['workspace_path'])
+    ])
+    subprocess.check_call([
+        'chmod', '+x',
+        "{}/test_server.bin".format(ctxt['workspace_path'])
+    ])
+
+
 def get_service_list(keystone_session):
     """Retrieve list of services from keystone.
 
@@ -227,7 +249,7 @@ def add_auth_config(ctxt):
             overcloud_auth['OS_PROJECT_DOMAIN_NAME'])
 
 
-def get_tempest_context():
+def get_tempest_context(workspace_path):
     """Generate the tempest config context.
 
     :returns: Context dictionary
@@ -235,6 +257,7 @@ def get_tempest_context():
     """
     keystone_session = openstack_utils.get_overcloud_keystone_session()
     ctxt = {}
+    ctxt['workspace_path'] = workspace_path
     ctxt_funcs = {
         'nova': add_nova_config,
         'neutron': add_neutron_config,
@@ -253,6 +276,8 @@ def get_tempest_context():
             ctxt_func(ctxt, keystone_session)
     add_environment_var_config(ctxt, ctxt['enabled_services'])
     add_auth_config(ctxt)
+    if 'octavia' in ctxt['enabled_services']:
+        add_octavia_config(ctxt)
     return ctxt
 
 
@@ -289,13 +314,14 @@ def setup_tempest(tempest_template, accounts_template):
     workspace_name, workspace_path = tempest_utils.get_workspace()
     tempest_utils.destroy_workspace(workspace_name, workspace_path)
     tempest_utils.init_workspace(workspace_path)
+    context = get_tempest_context(workspace_path)
     render_tempest_config(
         os.path.join(workspace_path, 'etc/tempest.conf'),
-        get_tempest_context(),
+        context,
         tempest_template)
     render_tempest_config(
         os.path.join(workspace_path, 'etc/accounts.yaml'),
-        get_tempest_context(),
+        context,
         accounts_template)
 
 

zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2

@@ -4,7 +4,6 @@ use_stderr = false
 log_file = tempest.log
 
 [auth]
-test_accounts_file = accounts.yaml
 default_credentials_domain_name = {{ default_credentials_domain_name }}
 admin_username = {{ admin_username }}
 admin_project_name = {{ admin_project_name }}
@@ -65,6 +64,7 @@ floating_network_name = {{ ext_net }}
 [network-feature-enabled]
 ipv6 = false
 api_extensions = {{ neutron_api_extensions }}
+port_security = true
 {% endif %}
 
 {% if 'heat' in enabled_services %}
@@ -104,3 +104,10 @@ catalog_type = {{ catalog_type }}
 [volume-feature-enabled]
 backup = false
 {% endif %}
+
+{% if 'octavia' in enabled_services %}
+[load_balancer]
+enable_security_groups = true
+test_with_ipv6 = false
+test_server_path = {{ workspace_path }}/test_server.bin
+{% endif %}

zaza/openstack/charm_tests/test_utils.py

@@ -18,6 +18,7 @@ import subprocess
 import sys
 import tenacity
 import unittest
+import yaml
 
 import novaclient
 
@@ -120,22 +121,102 @@ class BaseCharmTest(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls, application_name=None, model_alias=None):
-        """Run setup for test class to create common resources."""
+        """Run setup for test class to create common resources.
+
+        Note: the derived class may not use the application_name; if it's set
+        to None then this setUpClass() method will attempt to extract the
+        application name from the charm_config (essentially the test.yaml)
+        using the key 'charm_name' in the test_config.  If that isn't present,
+        then there will be no application_name set, and this is considered a
+        generic scenario of a whole model rather than a particular charm under
+        test.
+
+        :param application_name: the name of the applications that the derived
+            class is testing.  If None, then it's a generic test not connected
+            to any single charm.
+        :type application_name: Optional[str]
+        :param model_alias: the alias to use if needed.
+        :type model_alias: Optional[str]
+        """
         cls.model_aliases = model.get_juju_model_aliases()
         if model_alias:
             cls.model_name = cls.model_aliases[model_alias]
         else:
             cls.model_name = model.get_juju_model()
         cls.test_config = lifecycle_utils.get_charm_config(fatal=False)
+
         if application_name:
             cls.application_name = application_name
         else:
-            cls.application_name = cls.test_config['charm_name']
+            try:
+                charm_under_test_name = cls.test_config['charm_name']
+            except KeyError:
+                logging.warning("No application_name and no charm config so "
+                                "not setting the application_name. Likely a "
+                                "scenario test.")
+                return
+            deployed_app_names = model.sync_deployed(model_name=cls.model_name)
+            if charm_under_test_name in deployed_app_names:
+                # There is an application named like the charm under test.
+                # Let's consider it the application under test:
+                cls.application_name = charm_under_test_name
+            else:
+                # Let's search for any application whose name starts with the
+                # name of the charm under test and assume it's the application
+                # under test:
+                for app_name in deployed_app_names:
+                    if app_name.startswith(charm_under_test_name):
+                        cls.application_name = app_name
+                        break
+                else:
+                    logging.warning('Could not find application under test')
+                    return
+
         cls.lead_unit = model.get_lead_unit_name(
             cls.application_name,
             model_name=cls.model_name)
         logging.debug('Leader unit is {}'.format(cls.lead_unit))
 
+    def config_current_separate_non_string_type_keys(
+            self, non_string_type_keys, config_keys=None,
+            application_name=None):
+        """Obtain current config and the non-string type config separately.
+
+        If the charm config option is not string, it will not accept being
+        reverted back in "config_change()" method if the current value is None.
+        Therefore, obtain the current config and separate those out, so they
+        can be used for a separate invocation of "config_change()" with
+        reset_to_charm_default set to True.
+
+        :param config_keys: iterable of strs to index into the current config.
+                            If None, return all keys from the config
+        :type config_keys:  Optional[Iterable[str]]
+        :param non_string_type_keys: list of non-string type keys to be
+                                     separated out only if their current value
+                                     is None
+        :type non_string_type_keys: list
+        :param application_name: String application name for use when called
+                                 by a charm under test other than the object's
+                                 application.
+        :type application_name:  Optional[str]
+        :return: Dictionary of current charm configs without the
+                 non-string type keys provided, and dictionary of the
+                 non-string keys found in the supplied config_keys list.
+        :rtype: Dict[str, Any], Dict[str, None]
+        """
+        current_config = self.config_current(application_name, config_keys)
+        non_string_type_config = {}
+        if config_keys is None:
+            config_keys = list(current_config.keys())
+        for key in config_keys:
+            # We only care if the current value is None, otherwise it will
+            # not face issues being reverted by "config_change()"
+            if key in non_string_type_keys and current_config[key] is None:
+                non_string_type_config[key] = None
+                current_config.pop(key)
+
+        return current_config, non_string_type_config
+
     def config_current(self, application_name=None, keys=None):
         """Get Current Config of an application normalized into key-values.
 
@@ -257,7 +338,7 @@ class BaseCharmTest(unittest.TestCase):
                           'charm default: "{}"'
                           .format(alternate_config.keys()))
             model.reset_application_config(application_name,
-                                           alternate_config.keys(),
+                                           list(alternate_config.keys()),
                                            model_name=self.model_name)
         elif default_config == alternate_config:
             logging.debug('default_config == alternate_config, not attempting '
@@ -492,6 +573,32 @@ class BaseCharmTest(unittest.TestCase):
         return self.test_config.get('tests_options', {}).get(
             '.'.join(caller_path + [key]), default)
 
+    def get_applications_with_substring_in_name(self, substring):
+        """Get applications with substring in name.
+
+        :param substring: String to search for in application names
+        :type substring: str
+        :returns: List of matching applictions
+        :rtype: List
+        """
+        status = model.get_status().applications
+        applications = []
+        for application in status.keys():
+            if substring in application:
+                applications.append(application)
+        return applications
+
+    def run_update_status_hooks(self, units):
+        """Run update status hooks on units.
+
+        :param units: List of unit names or unit.entity_id
+        :type units: List[str]
+        :returns: None
+        :rtype: None
+        """
+        for unit in units:
+            model.run_on_unit(unit, "hooks/update-status")
+
 
 class OpenStackBaseTest(BaseCharmTest):
     """Generic helpers for testing OpenStack API charms."""
@@ -526,7 +633,8 @@ class OpenStackBaseTest(BaseCharmTest):
             # Test did not define self.RESOURCE_PREFIX, ignore.
             pass
 
-    def launch_guest(self, guest_name, userdata=None):
+    def launch_guest(self, guest_name, userdata=None, use_boot_volume=False,
+                     instance_key=None):
         """Launch two guests to use in tests.
 
         Note that it is up to the caller to have set the RESOURCE_PREFIX class
@@ -539,25 +647,38 @@ class OpenStackBaseTest(BaseCharmTest):
         :type guest_name: str
         :param userdata: Userdata to attach to instance
         :type userdata: Optional[str]
+        :param use_boot_volume: Whether to boot guest from a shared volume.
+        :type use_boot_volume: boolean
+        :param instance_key: Key to collect associated config data with.
+        :type instance_key: Optional[str]
         :returns: Nova instance objects
         :rtype: Server
         """
+        instance_key = instance_key or glance_setup.LTS_IMAGE_NAME
         instance_name = '{}-{}'.format(self.RESOURCE_PREFIX, guest_name)
 
-        instance = self.retrieve_guest(instance_name)
-        if instance:
-            logging.info('Removing already existing instance ({}) with '
-                         'requested name ({})'
-                         .format(instance.id, instance_name))
-            openstack_utils.delete_resource(
-                self.nova_client.servers,
-                instance.id,
-                msg="server")
+        for attempt in tenacity.Retrying(
+                stop=tenacity.stop_after_attempt(3),
+                wait=tenacity.wait_exponential(
+                    multiplier=1, min=2, max=10)):
+            with attempt:
+                old_instance_with_same_name = self.retrieve_guest(
+                    instance_name)
+                if old_instance_with_same_name:
+                    logging.info(
+                        'Removing already existing instance ({}) with '
+                        'requested name ({})'
+                        .format(old_instance_with_same_name.id, instance_name))
+                    openstack_utils.delete_resource(
+                        self.nova_client.servers,
+                        old_instance_with_same_name.id,
+                        msg="server")
 
-        return configure_guest.launch_instance(
-            glance_setup.LTS_IMAGE_NAME,
-            vm_name=instance_name,
-            userdata=userdata)
+                return configure_guest.launch_instance(
+                    instance_key,
+                    vm_name=instance_name,
+                    use_boot_volume=use_boot_volume,
+                    userdata=userdata)
 
     def launch_guests(self, userdata=None):
         """Launch two guests to use in tests.
@@ -572,15 +693,10 @@ class OpenStackBaseTest(BaseCharmTest):
         """
         launched_instances = []
         for guest_number in range(1, 2+1):
-            for attempt in tenacity.Retrying(
-                    stop=tenacity.stop_after_attempt(3),
-                    wait=tenacity.wait_exponential(
-                        multiplier=1, min=2, max=10)):
-                with attempt:
-                    launched_instances.append(
-                        self.launch_guest(
-                            guest_name='ins-{}'.format(guest_number),
-                            userdata=userdata))
+            launched_instances.append(
+                self.launch_guest(
+                    guest_name='ins-{}'.format(guest_number),
+                    userdata=userdata))
         return launched_instances
 
     def retrieve_guest(self, guest_name):
@@ -612,3 +728,400 @@ class OpenStackBaseTest(BaseCharmTest):
         instance_2 = self.retrieve_guest(
             '{}-ins-1'.format(self.RESOURCE_PREFIX))
         return instance_1, instance_2
+
+
+class BaseDeferredRestartTest(BaseCharmTest):
+    """Check deferred restarts.
+
+    Example of adding a deferred restart test::
+
+        class NeutronOVSDeferredRestartTest(
+            test_utils.BaseDeferredRestartTest):
+
+            @classmethod
+            def setUpClass(cls):
+                super().setUpClass(application_name='neutron-openvswitch')
+
+            def run_tests(self):
+                # Trigger a config change which triggers a deferred hook.
+                self.run_charm_change_hook_test('config-changed')
+
+                # Trigger a package change which requires a restart
+                self.run_package_change_test(
+                    'openvswitch-switch',
+                    'openvswitch-switch')
+
+
+    NOTE: The test has been broken into various class methods which may require
+          specialisation if the charm being tested is not a standard OpenStack
+          charm e.g. `trigger_deferred_hook_via_charm` if the charm is not
+          an oslo config or does not have a debug option.
+    """
+
+    @classmethod
+    def setUpClass(cls, application_name):
+        """Run test setup.
+
+        :param application_name: Name of application to run tests against.
+        :type application_name: str
+        """
+        cls.application_name = application_name
+        super().setUpClass(application_name=cls.application_name)
+
+    def check_status_message_is_clear(self):
+        """Check each units status message show no defeerred events."""
+        # Check workload status no longer shows deferred restarts.
+        for unit in model.get_units(self.application_name):
+            model.block_until_unit_wl_message_match(
+                unit.entity_id,
+                'Unit is ready')
+        model.block_until_all_units_idle()
+
+    def check_clear_restarts(self):
+        """Clear and deferred restarts and check status.
+
+        Clear and deferred restarts and then check the workload status message
+        for each unit.
+        """
+        # Use action to run any deferred restarts
+        for unit in model.get_units(self.application_name):
+            logging.info("Running restart-services on {}".format(
+                unit.entity_id))
+            model.run_action(
+                unit.entity_id,
+                'restart-services',
+                action_params={'deferred-only': True},
+                raise_on_failure=True)
+
+        # Check workload status no longer shows deferred restarts.
+        self.check_status_message_is_clear()
+
+    def clear_hooks(self):
+        """Clear and deferred hooks.
+
+        Run any deferred hooks.
+        """
+        # Use action to run any deferred restarts
+        for unit in model.get_units(self.application_name):
+            logging.info("Running run-deferred-hooks on {}".format(
+                unit.entity_id))
+            model.run_action(
+                unit.entity_id,
+                'run-deferred-hooks',
+                raise_on_failure=True)
+
+    def check_clear_hooks(self):
+        """Clear deferred hooks and check status.
+
+        Clear deferred hooks and then check the workload status message
+        for each unit.
+        """
+        self.clear_hooks()
+        # Check workload status no longer shows deferred restarts.
+        self.check_status_message_is_clear()
+
+    def run_show_deferred_events_action(self):
+        """Run show-deferred-events and return results.
+
+        :returns: Data from action run
+        :rtype: Dict
+        """
+        unit = model.get_units(self.application_name)[0]
+        action = model.run_action(
+            unit.entity_id,
+            'show-deferred-events',
+            raise_on_failure=True)
+        return yaml.safe_load(action.data['results']['output'])
+
+    def check_show_deferred_events_action_restart(self, test_service,
+                                                  restart_reason):
+        """Check the output from the action to list deferred restarts.
+
+        Run the action to list any deferred restarts and check it has entry for
+        the given service and reason.
+
+        :param test_service: Service that should need a restart
+        :type test_service: str
+        :param restart_reason: The reason the action should list for the
+                               service needing to be restarted. This can be a
+                               substring.
+        :type restart_reason: str
+        """
+        # Ensure that the deferred restart and cause are listed via action
+        logging.info(
+            ("Checking {} is marked as needing restart in "
+             "show-deferred-events action").format(
+                test_service))
+        for event in self.run_show_deferred_events_action()['restarts']:
+            logging.info("{} in {} and {} in {}".format(
+                test_service,
+                event,
+                restart_reason,
+                event))
+            if test_service in event and restart_reason in event:
+                break
+        else:
+            msg = 'No entry for restart of {} for reason {} found'.format(
+                test_service,
+                restart_reason)
+            raise Exception(msg)
+
+    def check_show_deferred_events_action_hook(self, hook):
+        """Check the output from the action to list deferred eveents.
+
+        Run the action to list any deferred events and check it has entry for
+        the given hook.
+
+        :param hook: Hook or method name
+        :type hook: str
+        """
+        # Ensure that the deferred restart and cause are listed via action
+        logging.info(
+            ("Checking {} is marked as skipped in "
+             "show-deferred-events action").format(hook))
+        for event in self.run_show_deferred_events_action()['hooks']:
+            logging.info("{} in {}".format(hook, event))
+            if hook in event:
+                break
+        else:
+            msg = '{} not found in {}'.format(hook, event)
+            raise Exception(msg)
+
+    def check_show_deferred_restarts_wlm(self, test_service):
+        """Check the workload status message lists deferred restart.
+
+        :param test_service: Service that should need a restart
+        :type test_service: str
+        """
+        # Ensure that the deferred restarts are visible in Juju status
+        for unit in model.get_units(self.application_name):
+            # Just checking one example service should we be checking all?
+            logging.info(
+                ("Checking {} is marked as needing restart in workload "
+                 "message of {}".format(test_service, unit.entity_id)))
+            assert test_service in unit.workload_status_message
+
+    def check_deferred_hook_wlm(self, deferred_hook):
+        """Check the workload status message lists deferred event.
+
+        :param deferred_hook: Hook or method name which should be showing as
+                              deferred.
+        :type deferred_hook: str
+        """
+        # Ensure that the deferred restarts are visible in Juju status
+        for unit in model.get_units(self.application_name):
+            logging.info(
+                ("Checking {} is marked as having deferred hook in workload "
+                 "message".format(unit.entity_id)))
+            assert deferred_hook in unit.workload_status_message
+
+    def get_new_config(self):
+        """Return the config key and new value to trigger a hook execution.
+
+        NOTE: The implementation assumes the charm has a `debug` option and
+              If that is not true the derived class should override this
+              method.
+        :returns: Config key and new value
+        :rtype: (str, bool)
+        """
+        app_config = model.get_application_config(self.application_name)
+        return 'debug', str(not app_config['debug']['value'])
+
+    def set_new_config(self):
+        """Change applications charm config."""
+        logging.info("Triggering deferred restart via config change")
+        config_key, new_value = self.get_new_config()
+        logging.info("Setting {}: {}".format(config_key, new_value))
+        model.set_application_config(
+            self.application_name,
+            {config_key: new_value})
+        return new_value
+
+    def trigger_deferred_restart_via_charm(self, restart_config_file):
+        """Set charm config option which requires a service start.
+
+        Set the charm debug option and wait for that change to be renderred in
+        applications config file.
+
+        NOTE: The implementation assumes the restart_config_file in an oslo
+              config file. If that is not true the derived class should
+              override this method.
+
+        :param restart_config_file: Config file that updated value is expected
+                                    in.
+        :type restart_config_file: str
+        """
+        new_debug_value = self.set_new_config()
+        expected_contents = {
+            'DEFAULT': {
+                'debug': [new_debug_value]}}
+        logging.info("Waiting for debug to be {} in {}".format(
+            new_debug_value,
+            restart_config_file))
+        model.block_until_oslo_config_entries_match(
+            self.application_name,
+            restart_config_file,
+            expected_contents)
+        logging.info("Waiting for units to be idle")
+        model.block_until_all_units_idle()
+
+    def trigger_deferred_hook_via_charm(self, deferred_hook):
+        """Set charm config option which requires a service start.
+
+        Set the charm debug option and wait for that change to be rendered in
+        applications config file.
+
+        :param deferred_hook: Hook or method name which should be showing as
+                              deferred.
+        :type deferred_hook: str
+        :returns: New config value
+        :rtype: Union[str, int, float]
+        """
+        new_debug_value = self.set_new_config()
+        for unit in model.get_units(self.application_name):
+            logging.info('Waiting for {} to show deferred hook'.format(
+                unit.entity_id))
+            model.block_until_unit_wl_message_match(
+                unit.entity_id,
+                status_pattern='.*{}.*'.format(deferred_hook))
+        logging.info("Waiting for units to be idle")
+        model.block_until_all_units_idle()
+        return new_debug_value
+
+    def trigger_deferred_restart_via_package(self, restart_package):
+        """Update a package which requires a service restart.
+
+        :param restart_package: Package that will be changed to trigger a
+                                service restart.
+        :type restart_package: str
+        """
+        logging.info("Triggering deferred restart via package change")
+        # Test restart requested by package
+        for unit in model.get_units(self.application_name):
+            model.run_on_unit(
+                unit.entity_id,
+                ('dpkg-reconfigure {}; '
+                 'JUJU_HOOK_NAME=update-status ./hooks/update-status').format(
+                    restart_package))
+
+    def run_charm_change_restart_test(self, test_service, restart_config_file):
+        """Trigger a deferred restart by updating a config file via the charm.
+
+        Trigger a hook in the charm which the charm will defer.
+
+        :param test_service: Service that should need a restart
+        :type test_service: str
+        :param restart_config_file: Config file that updated value is expected
+                                    in.
+        :type restart_config_file: str
+        """
+        self.trigger_deferred_restart_via_charm(restart_config_file)
+
+        self.check_show_deferred_restarts_wlm(test_service)
+        self.check_show_deferred_events_action_restart(
+            test_service,
+            restart_config_file)
+        logging.info("Running restart action to clear deferred restarts")
+        self.check_clear_restarts()
+
+    def run_charm_change_hook_test(self, deferred_hook):
+        """Trigger a deferred restart by updating a config file via the charm.
+
+        :param deferred_hook: Hook or method name which should be showing as
+                              defeerred.
+        :type deferred_hook: str
+        """
+        self.trigger_deferred_hook_via_charm(deferred_hook)
+
+        self.check_deferred_hook_wlm(deferred_hook)
+        self.check_show_deferred_events_action_hook(deferred_hook)
+        # Rerunning to flip config option back to previous value.
+        self.trigger_deferred_hook_via_charm(deferred_hook)
+        logging.info("Running restart action to clear deferred hooks")
+        # If there are a number of units in the application and restarts take
+        # time then another deferred hook can occur so do not block on a
+        # clear status message.
+        self.clear_hooks()
+
+    def run_package_change_test(self, restart_package, restart_package_svc):
+        """Trigger a deferred restart by updating a package.
+
+        Update a package which requires will add a deferred restart.
+
+        :param restart_package: Package that will be changed to trigger a
+                                service restart.
+        :type restart_package: str
+        :param restart_package_service: Service that will require a restart
+                                        after restart_package has changed.
+        :type restart_package_service: str
+        """
+        self.trigger_deferred_restart_via_package(restart_package)
+
+        self.check_show_deferred_restarts_wlm(restart_package_svc)
+        self.check_show_deferred_events_action_restart(
+            restart_package_svc,
+            'Package update')
+        logging.info("Running restart action to clear deferred restarts")
+        self.check_clear_restarts()
+
+    def run_tests(self):
+        """Run charm tests. should specify which tests to run.
+
+        The charm test that implements this test should specify which tests to
+        run, for example:
+
+            def run_tests(self):
+                # Trigger a config change which triggers a deferred hook.
+                self.run_charm_change_hook_test('config-changed')
+
+                # Trigger a config change which requires a restart
+                self.run_charm_change_restart_test(
+                    'neutron-l3-agent',
+                    '/etc/neutron/neutron.conf')
+
+                # Trigger a package change which requires a restart
+                self.run_package_change_test(
+                    'openvswitch-switch',
+                    'openvswitch-switch')
+        """
+        raise NotImplementedError
+
+    def test_deferred_restarts(self):
+        """Run deferred restart tests."""
+        app_config = model.get_application_config(self.application_name)
+        auto_restart_config_key = 'enable-auto-restarts'
+        if auto_restart_config_key not in app_config:
+            raise unittest.SkipTest("Deferred restarts not implemented")
+
+        # Ensure auto restarts are off.
+        policy_file = '/etc/policy-rc.d/charm-{}.policy'.format(
+            self.application_name)
+        if app_config[auto_restart_config_key]['value']:
+            logging.info("Turning off auto restarts")
+            model.set_application_config(
+                self.application_name, {auto_restart_config_key: 'False'})
+            logging.info("Waiting for {} to appear on units of {}".format(
+                policy_file,
+                self.application_name))
+            model.block_until_file_has_contents(
+                self.application_name,
+                policy_file,
+                'policy_requestor_name')
+            # The block_until_file_has_contents ensures the change we waiting
+            # for has happened, now just wait for any hooks to finish.
+            logging.info("Waiting for units to be idle")
+            model.block_until_all_units_idle()
+        else:
+            logging.info("Auto restarts already disabled")
+
+        self.run_tests()
+
+        # Finished so turn auto-restarts back on.
+        logging.info("Turning on auto restarts")
+        model.set_application_config(
+            self.application_name, {auto_restart_config_key: 'True'})
+        model.block_until_file_missing(
+            self.application_name,
+            policy_file)
+        model.block_until_all_units_idle()
+        self.check_clear_hooks()

zaza/openstack/charm_tests/trilio/setup.py

@@ -19,12 +19,16 @@
 import logging
 import os
 
+import boto3
+
+import zaza.charm_lifecycle.utils as lifecycle_utils
 import zaza.model as zaza_model
 import zaza.openstack.utilities.juju as juju_utils
 import zaza.openstack.utilities.generic as generic_utils
+import zaza.openstack.utilities.openstack as openstack_utils
 
 
-def basic_setup():
+def nfs_setup():
     """Run setup for testing Trilio.
 
     Setup for testing Trilio is currently part of functional
@@ -35,12 +39,14 @@ def basic_setup():
     trilio_wlm_unit = zaza_model.get_first_unit_name("trilio-wlm")
 
     nfs_shares_conf = {"nfs-shares": "{}:/srv/testing".format(nfs_server_ip)}
+    logging.info("NFS share config: {}".format(nfs_shares_conf))
     _trilio_services = ["trilio-wlm", "trilio-data-mover"]
 
     conf_changed = False
     for juju_service in _trilio_services:
         app_config = zaza_model.get_application_config(juju_service)
         if app_config["nfs-shares"] != nfs_shares_conf["nfs-shares"]:
+            logging.info("Updating nfs-shares config option")
             zaza_model.set_application_config(juju_service, nfs_shares_conf)
             conf_changed = True
 
@@ -54,6 +60,9 @@ def basic_setup():
             target_status="active",
         )
 
+
+def trust_setup():
+    """Run setup Trilio trust setup."""
     logging.info("Executing create-cloud-admin-trust")
     password = juju_utils.leader_get("keystone", "admin_passwd")
 
@@ -66,6 +75,9 @@ def basic_setup():
         )
     )
 
+
+def license_setup():
+    """Run setup Trilio license setup."""
     logging.info("Executing create-license")
     test_license = os.environ.get("TEST_TRILIO_LICENSE")
     if test_license and os.path.exists(test_license):
@@ -81,3 +93,86 @@ def basic_setup():
 
     else:
         logging.error("Unable to find Trilio License file")
+
+
+def s3_setup():
+    """Run setup of s3 options for Trilio."""
+    session = openstack_utils.get_overcloud_keystone_session()
+    ks_client = openstack_utils.get_keystone_session_client(
+        session)
+
+    # Get token data so we can glean our user_id and project_id
+    token_data = ks_client.tokens.get_token_data(session.get_token())
+    project_id = token_data['token']['project']['id']
+    user_id = token_data['token']['user']['id']
+
+    # Store URL to service providing S3 compatible API
+    for entry in token_data['token']['catalog']:
+        if entry['type'] == 's3':
+            for endpoint in entry['endpoints']:
+                if endpoint['interface'] == 'public':
+                    s3_region = endpoint['region']
+                    s3_endpoint = endpoint['url']
+
+    # Create AWS compatible application credentials in Keystone
+    ec2_creds = ks_client.ec2.create(user_id, project_id)
+    cacert = openstack_utils.get_cacert()
+    kwargs = {
+        'region_name': s3_region,
+        'aws_access_key_id': ec2_creds.access,
+        'aws_secret_access_key': ec2_creds.secret,
+        'endpoint_url': s3_endpoint,
+        'verify': cacert,
+    }
+    s3 = boto3.resource('s3', **kwargs)
+
+    # Create bucket
+    bucket_name = 'zaza-trilio'
+    logging.info("Creating bucket: {}".format(bucket_name))
+    bucket = s3.Bucket(bucket_name)
+    bucket.create()
+
+    s3_config = {
+        'tv-s3-secret-key': ec2_creds.secret,
+        'tv-s3-access-key': ec2_creds.access,
+        'tv-s3-region-name': s3_region,
+        'tv-s3-bucket': bucket_name,
+        'tv-s3-endpoint-url': s3_endpoint}
+    for app in ['trilio-wlm', 'trilio-data-mover']:
+        logging.info("Setting s3 config for {}".format(app))
+        zaza_model.set_application_config(app, s3_config)
+    test_config = lifecycle_utils.get_charm_config(fatal=False)
+    states = test_config.get('target_deploy_status', {})
+    states['trilio-wlm'] = {
+        'workload-status': 'blocked',
+        'workload-status-message': 'application not trusted'}
+    zaza_model.wait_for_application_states(
+        states=test_config.get('target_deploy_status', {}),
+        timeout=7200)
+    zaza_model.block_until_all_units_idle()
+
+
+def basic_setup():
+    """Run basic setup for Trilio apps."""
+    backup_target_type = zaza_model.get_application_config(
+        'trilio-wlm')['backup-target-type']['value']
+    if backup_target_type == "nfs":
+        nfs_setup()
+    if backup_target_type in ["s3", "experimental-s3"]:
+        s3_setup()
+    trust_setup()
+    license_setup()
+
+
+def python2_workaround():
+    """Workaround for Bug #1915914.
+
+    Trilio code currently has a bug which assumes an executable called 'python'
+    will be on the path. To workaround this install a package which adds a
+    symlink to python
+    """
+    for unit in zaza_model.get_units('trilio-wlm'):
+        zaza_model.run_on_unit(
+            unit.entity_id,
+            ("apt install --yes python-is-python3; "
+             "systemctl restart wlm\\*.service"))

zaza/openstack/charm_tests/trilio/tests.py

@@ -262,7 +262,7 @@ class WorkloadmgrCLIHelper(object):
 
         retryer = tenacity.Retrying(
             wait=tenacity.wait_exponential(multiplier=1, max=30),
-            stop=tenacity.stop_after_delay(720),
+            stop=tenacity.stop_after_delay(900),
             reraise=True,
         )
 
@@ -440,7 +440,7 @@ class TrilioGhostNFSShareTest(TrilioBaseTest):
         )
 
 
-class TrilioWLMTest(TrilioGhostNFSShareTest):
+class TrilioWLMBaseTest(TrilioBaseTest):
     """Tests for Trilio Workload Manager charm."""
 
     conf_file = "/etc/workloadmgr/workloadmgr.conf"
@@ -463,10 +463,26 @@ class TrilioDMAPITest(TrilioBaseTest):
     services = ["dmapi-api"]
 
 
-class TrilioDataMoverTest(TrilioGhostNFSShareTest):
+class TrilioDataMoverBaseTest(TrilioBaseTest):
     """Tests for Trilio Data Mover charm."""
 
     conf_file = "/etc/tvault-contego/tvault-contego.conf"
     application_name = "trilio-data-mover"
 
     services = ["tvault-contego"]
+
+
+class TrilioDataMoverNFSTest(TrilioDataMoverBaseTest, TrilioGhostNFSShareTest):
+    """Tests for Trilio Data Mover charm backed by NFS."""
+
+
+class TrilioDataMoverS3Test(TrilioDataMoverBaseTest):
+    """Tests for Trilio Data Mover charm backed by S3."""
+
+
+class TrilioWLMNFSTest(TrilioWLMBaseTest, TrilioGhostNFSShareTest):
+    """Tests for Trilio WLM charm backed by NFS."""
+
+
+class TrilioWLMS3Test(TrilioWLMBaseTest):
+    """Tests for Trilio WLM charm backed by S3."""

zaza/openstack/charm_tests/vault/setup.py

@@ -26,6 +26,7 @@ import zaza.model
 import zaza.openstack.utilities.cert
 import zaza.openstack.utilities.openstack
 import zaza.openstack.utilities.generic
+import zaza.openstack.utilities.exceptions as zaza_exceptions
 import zaza.utilities.juju as juju_utils
 
 
@@ -73,9 +74,27 @@ def basic_setup_and_unseal(cacert=None):
         zaza.model.run_on_unit(unit.name, './hooks/update-status')
 
 
+async def mojo_or_default_unseal_by_unit():
+    """Unseal any units reported as sealed using a cacert.
+
+    The mojo cacert is tried first, and if that doesn't exist, then the default
+    zaza located cacert is used.
+    """
+    try:
+        await mojo_unseal_by_unit()
+    except zaza_exceptions.CACERTNotFound:
+        await unseal_by_unit()
+
+
 def mojo_unseal_by_unit():
     """Unseal any units reported as sealed using mojo cacert."""
     cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path()
+    unseal_by_unit(cacert)
+
+
+def unseal_by_unit(cacert=None):
+    """Unseal any units reported as sealed using mojo cacert."""
+    cacert = cacert or get_cacert_file()
     vault_creds = vault_utils.get_credentails()
     for client in vault_utils.get_clients(cacert=cacert):
         if client.hvac_client.is_sealed():
@@ -86,9 +105,27 @@ def mojo_unseal_by_unit():
             zaza.model.run_on_unit(unit_name, './hooks/update-status')
 
 
+async def async_mojo_or_default_unseal_by_unit():
+    """Unseal any units reported as sealed using a cacert.
+
+    The mojo cacert is tried first, and if that doesn't exist, then the default
+    zaza located cacert is used.
+    """
+    try:
+        await async_mojo_unseal_by_unit()
+    except zaza_exceptions.CACERTNotFound:
+        await async_unseal_by_unit()
+
+
 async def async_mojo_unseal_by_unit():
     """Unseal any units reported as sealed using mojo cacert."""
     cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path()
+    await async_unseal_by_unit(cacert)
+
+
+async def async_unseal_by_unit(cacert=None):
+    """Unseal any units reported as sealed using vault cacert."""
+    cacert = cacert or get_cacert_file()
     vault_creds = vault_utils.get_credentails()
     for client in vault_utils.get_clients(cacert=cacert):
         if client.hvac_client.is_sealed():
@@ -137,7 +174,8 @@ def auto_initialize(cacert=None, validation_application='keystone', wait=True):
         zaza.model.wait_for_agent_status()
         test_config = lifecycle_utils.get_charm_config(fatal=False)
         zaza.model.wait_for_application_states(
-            states=test_config.get('target_deploy_status', {}))
+            states=test_config.get('target_deploy_status', {}),
+            timeout=7200)
 
     if validation_application:
         validate_ca(cacertificate, application=validation_application)
@@ -184,9 +222,8 @@ def validate_ca(cacertificate, application="keystone", port=5000):
     :returns: None
     :rtype: None
     """
-    zaza.model.block_until_file_has_contents(
+    zaza.openstack.utilities.openstack.block_until_ca_exists(
         application,
-        zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT,
         cacertificate.decode().strip())
     vip = (zaza.model.get_application_config(application)
            .get("vip").get("value"))

zaza/openstack/charm_tests/vault/tests.py

@@ -18,11 +18,13 @@
 
 import contextlib
 import hvac
+import json
 import logging
 import time
 import unittest
 import uuid
 import tempfile
+import tenacity
 
 import requests
 import zaza.charm_lifecycle.utils as lifecycle_utils
@@ -31,6 +33,7 @@ import zaza.openstack.charm_tests.vault.utils as vault_utils
 import zaza.openstack.utilities.cert
 import zaza.openstack.utilities.openstack
 import zaza.model
+import zaza.utilities.juju as juju_utils
 
 
 class BaseVaultTest(test_utils.OpenStackBaseTest):
@@ -110,7 +113,11 @@ class UnsealVault(BaseVaultTest):
         vault_utils.run_charm_authorize(self.vault_creds['root_token'])
         if not test_config:
             test_config = lifecycle_utils.get_charm_config()
-        del test_config['target_deploy_status']['vault']
+        try:
+            del test_config['target_deploy_status']['vault']
+        except KeyError:
+            # Already removed
+            pass
         zaza.model.wait_for_application_states(
             states=test_config.get('target_deploy_status', {}))
 
@@ -153,19 +160,31 @@ class VaultTest(BaseVaultTest):
             allowed_domains='openstack.local')
 
         test_config = lifecycle_utils.get_charm_config()
-        del test_config['target_deploy_status']['vault']
-        zaza.model.block_until_file_has_contents(
+        try:
+            del test_config['target_deploy_status']['vault']
+        except KeyError:
+            # Already removed
+            pass
+        zaza.openstack.utilities.openstack.block_until_ca_exists(
             'keystone',
-            zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT,
             cacert.decode().strip())
         zaza.model.wait_for_application_states(
             states=test_config.get('target_deploy_status', {}))
         ip = zaza.model.get_app_ips(
             'keystone')[0]
+
         with tempfile.NamedTemporaryFile(mode='w') as fp:
             fp.write(cacert.decode())
             fp.flush()
-            requests.get('https://{}:5000'.format(ip), verify=fp.name)
+            # Avoid race condition and retry
+            for attempt in tenacity.Retrying(
+                stop=tenacity.stop_after_attempt(3),
+                wait=tenacity.wait_exponential(
+                    multiplier=2, min=2, max=10)):
+                with attempt:
+                    logging.info(
+                        "Attempting to connect to https://{}:5000".format(ip))
+                    requests.get('https://{}:5000'.format(ip), verify=fp.name)
 
     def test_all_clients_authenticated(self):
         """Check all vault clients are authenticated."""
@@ -245,8 +264,6 @@ class VaultTest(BaseVaultTest):
         Pause service and check services are stopped, then resume and check
         they are started.
         """
-        # Restarting vault process will set it as sealed so it's
-        # important to have the test executed at the end.
         vault_actions = zaza.model.get_actions(
             'vault')
         if 'pause' not in vault_actions or 'resume' not in vault_actions:
@@ -258,6 +275,80 @@ class VaultTest(BaseVaultTest):
         lead_client = vault_utils.extract_lead_unit_client(self.clients)
         self.assertTrue(lead_client.hvac_client.seal_status['sealed'])
 
+    def test_vault_reload(self):
+        """Run reload tests.
+
+        Reload service and check services were restarted
+        by doing simple change in the running config by API.
+        Then confirm that service is not sealed
+        """
+        vault_actions = zaza.model.get_actions(
+            'vault')
+        if 'reload' not in vault_actions:
+            raise unittest.SkipTest("The version of charm-vault tested does "
+                                    "not have reload action")
+
+        container_results = zaza.model.run_on_leader(
+            "vault", "systemd-detect-virt --container"
+        )
+        container_rc = json.loads(container_results["Code"])
+        if container_rc == 0:
+            raise unittest.SkipTest(
+                "Vault unit is running in a container. Cannot use mlock."
+            )
+
+        lead_client = vault_utils.get_cluster_leader(self.clients)
+        running_config = vault_utils.get_running_config(lead_client)
+        value_to_set = not running_config['data']['disable_mlock']
+
+        logging.info("Setting disable-mlock to {}".format(str(value_to_set)))
+        zaza.model.set_application_config(
+            'vault',
+            {'disable-mlock': str(value_to_set)})
+
+        logging.info("Waiting for model to be idle ...")
+        zaza.model.block_until_all_units_idle(model_name=self.model_name)
+
+        logging.info("Testing action reload on {}".format(lead_client))
+        zaza.model.run_action(
+            juju_utils.get_unit_name_from_ip_address(
+                lead_client.addr, 'vault'),
+            'reload',
+            model_name=self.model_name)
+
+        logging.info("Getting new value ...")
+        new_value = vault_utils.get_running_config(lead_client)[
+            'data']['disable_mlock']
+
+        logging.info(
+            "Asserting new value {} is equal to set value {}"
+            .format(new_value, value_to_set))
+        self.assertEqual(
+            value_to_set,
+            new_value)
+
+        logging.info("Asserting not sealed")
+        self.assertFalse(lead_client.hvac_client.seal_status['sealed'])
+
+    def test_vault_restart(self):
+        """Run pause and resume tests.
+
+        Restart service and check services are started.
+        """
+        vault_actions = zaza.model.get_actions(
+            'vault')
+        if 'restart' not in vault_actions:
+            raise unittest.SkipTest("The version of charm-vault tested does "
+                                    "not have restart action")
+        logging.info("Testing restart")
+        zaza.model.run_action_on_leader(
+            'vault',
+            'restart',
+            action_params={})
+
+        lead_client = vault_utils.extract_lead_unit_client(self.clients)
+        self.assertTrue(lead_client.hvac_client.seal_status['sealed'])
+
 
 if __name__ == '__main__':
     unittest.main()

zaza/openstack/charm_tests/vault/utils.py

@@ -137,6 +137,41 @@ def get_vip_client(cacert=None):
     return client
 
 
+def get_cluster_leader(clients):
+    """Get Vault cluster leader.
+
+    We have to make sure we run api calls against the actual leader.
+
+    :param clients: Clients list to get leader
+    :type clients: List of CharmVaultClient
+    :returns: CharmVaultClient
+    :rtype: CharmVaultClient or None
+    """
+    if len(clients) == 1:
+        return clients[0]
+
+    for client in clients:
+        if client.hvac_client.ha_status['is_self']:
+            return client
+    return None
+
+
+def get_running_config(client):
+    """Get Vault running config.
+
+    The hvac library does not support getting info from endpoint
+    /v1/sys/config/state/sanitized Therefore we implement it here
+
+    :param client: Client used to get config
+    :type client: CharmVaultClient
+    :returns: dict from Vault api response
+    :rtype: dict
+    """
+    return requests.get(
+        client.hvac_client.adapter.base_uri + '/v1/sys/config/state/sanitized',
+        headers={'X-Vault-Token': client.hvac_client.token}).json()
+
+
 def init_vault(client, shares=1, threshold=1):
     """Initialise vault.
 

zaza/openstack/utilities/__init__.py

@@ -13,3 +13,155 @@
 # limitations under the License.
 
 """Collection of utilities to support zaza tests etc."""
+
+
+import time
+
+from keystoneauth1.exceptions.connection import ConnectFailure
+
+
+class ObjectRetrierWraps(object):
+    """An automatic retrier for an object.
+
+    This is designed to be used with an instance of an object.  Basically, it
+    wraps the object and any attributes that are fetched.  Essentially, it is
+    used to provide retries on method calls on openstack client objects in
+    tests to increase robustness of tests.
+
+    Although, technically this is bad, retries can be logged with the optional
+    log method.
+
+    Usage:
+
+        # get a client that does 3 retries, waits 5 seconds between retries and
+        # retries on any error.
+        some_client = ObjectRetrierWraps(get_some_client)
+        # this gets retried up to 3 times.
+        things = some_client.list_things()
+
+    Note, it is quite simple.  It wraps the object and on a getattr(obj, name)
+    it finds the name and then returns a wrapped version of that name.  On a
+    call, it returns the value of that call.  It only wraps objects in the
+    chain that are either callable or have a __getattr__() method.  i.e. one
+    that can then be retried or further fetched.  This means that if a.b.c() is
+    a chain of objects, and we just wrap 'a', then 'b' and 'c' will both be
+    wrapped that the 'c' object __call__() method will be the one that is
+    actually retried.
+
+    Note: this means that properties that do method calls won't be retried.
+    This is a limitation that may be addressed in the future, if it is needed.
+    """
+
+    def __init__(self, obj, num_retries=3, initial_interval=5.0, backoff=1.0,
+                 max_interval=15.0, total_wait=30.0, retry_exceptions=None,
+                 log=None):
+        """Initialise the retrier object.
+
+        :param obj: The object to wrap. Ought to be an instance of something
+            that you want to get methods on to call or be called itself.
+        :type obj: Any
+        :param num_retries: The (maximum) number of retries.  May not be hit if
+            the total_wait time is exceeded.
+        :type num_retries: int
+        :param initial_interval: The initial or starting interval between
+            retries.
+        :type initial_interval: float
+        :param backoff: The exponential backoff multiple.  1 is linear.
+        :type backoff: float
+        :param max_interval: The maximum interval between retries.
+            If backoff is >1 then the initial_interval will never grow larger
+            than max_interval.
+        :type max_interval: float
+        :param retry_exceptions: The list of exceptions to retry on, or None.
+            If a list, then it will only retry if the exception is one of the
+            ones in the list.
+        :type retry_exceptions: List[Exception]
+        """
+        # Note we use semi-private variable names that shouldn't clash with any
+        # on the actual object.
+        self.__obj = obj
+        self.__kwargs = {
+            'num_retries': num_retries,
+            'initial_interval': initial_interval,
+            'backoff': backoff,
+            'max_interval': max_interval,
+            'total_wait': total_wait,
+            'retry_exceptions': retry_exceptions,
+            'log': log or (lambda x: None),
+        }
+
+    def __getattr__(self, name):
+        """Get attribute; delegates to wrapped object."""
+        # Note the above may generate an attribute error; we expect this and
+        # will fail with an attribute error.
+        attr = getattr(self.__obj, name)
+        if callable(attr) or hasattr(attr, "__getattr__"):
+            return ObjectRetrierWraps(attr, **self.__kwargs)
+        else:
+            return attr
+        # TODO(ajkavanagh): Note detecting a property is a bit trickier.  we
+        # can do isinstance(attr, property), but then the act of accessing it
+        # is what calls it.  i.e. it would fail at the getattr(self.__obj,
+        # name) stage.  The solution is to check first, and if it's a property,
+        # then treat it like the retrier.  However, I think this is too
+        # complex for the first go, and to use manual retries in that instance.
+
+    def __call__(self, *args, **kwargs):
+        """Call the object; delegates to the wrapped object."""
+        obj = self.__obj
+        retry = 0
+        wait = self.__kwargs['initial_interval']
+        max_interval = self.__kwargs['max_interval']
+        log = self.__kwargs['log']
+        backoff = self.__kwargs['backoff']
+        total_wait = self.__kwargs['total_wait']
+        num_retries = self.__kwargs['num_retries']
+        retry_exceptions = self.__kwargs['retry_exceptions']
+        wait_so_far = 0
+        while True:
+            try:
+                return obj(*args, **kwargs)
+            except Exception as e:
+                # if retry_exceptions is not None, or the type of the exception
+                # is not in the list of retries, then raise an exception
+                # immediately.  This means that if retry_exceptions is None,
+                # then the method is always retried.
+                if (retry_exceptions is not None and
+                        type(e) not in retry_exceptions):
+                    raise
+                retry += 1
+                if retry > num_retries:
+                    log("{}: exceeded number of retries, so erroring out"
+                        .format(str(obj)))
+                    raise e
+                log("{}: call failed: retrying in {} seconds"
+                    .format(str(obj), wait))
+                time.sleep(wait)
+                wait_so_far += wait
+                if wait_so_far >= total_wait:
+                    raise e
+                wait = wait * backoff
+                if wait > max_interval:
+                    wait = max_interval
+
+
+def retry_on_connect_failure(client, **kwargs):
+    """Retry an object that eventually gets resolved to a call.
+
+    Specifically, this uses ObjectRetrierWraps but only against the
+    keystoneauth1.exceptions.connection.ConnectFailure exeception.
+
+    :params client: the object that may throw and exception when called.
+    :type client: Any
+    :params **kwargs: the arguments supplied to the ObjectRetrierWraps init
+                      method
+    :type **kwargs: Dict[Any]
+    :returns: client wrapped in an ObjectRetrierWraps instance
+    :rtype: ObjectRetrierWraps[client]
+    """
+    kwcopy = kwargs.copy()
+    if 'retry_exceptions' not in kwcopy:
+        kwcopy['retry_exceptions'] = []
+    if ConnectFailure not in kwcopy['retry_exceptions']:
+        kwcopy['retry_exceptions'].append(ConnectFailure)
+    return ObjectRetrierWraps(client, **kwcopy)

zaza/openstack/utilities/ceph.py

@@ -3,7 +3,7 @@ import json
 import logging
 
 import zaza.model as zaza_model
-import zaza.utilities.juju as zaza_juju
+import zaza.utilities.juju as juju_utils
 
 import zaza.openstack.utilities.openstack as openstack_utils
 
@@ -225,7 +225,7 @@ def get_pools_from_broker_req(application_or_unit, model_name=None):
     """
     # NOTE: we do not pass on a name for the remote_interface_name as that
     # varies between the Ceph consuming applications.
-    relation_data = zaza_juju.get_relation_from_unit(
+    relation_data = juju_utils.get_relation_from_unit(
         'ceph-mon', application_or_unit, None, model_name=model_name)
 
     # NOTE: we probably should consume the Ceph broker code from c-h but c-h is

zaza/openstack/utilities/generic.py

@@ -20,6 +20,7 @@ import os
 import socket
 import subprocess
 import telnetlib
+import tempfile
 import yaml
 
 from zaza import model
@@ -680,3 +681,28 @@ def get_mojo_cacert_path():
         return cacert
     else:
         raise zaza_exceptions.CACERTNotFound("Could not find cacert.pem")
+
+
+def attach_file_resource(application_name, resource_name,
+                         file_content, file_suffix=".txt"):
+    """Attaches a file as a Juju resource given the file content and suffix.
+
+    The file content will be written into a temporary file with the given
+    suffix, and it will be attached to the Juju application.
+
+    :param application_name: Juju application name.
+    :type application_name: string
+    :param resource_name: Juju resource name.
+    :type resource_name: string
+    :param file_content: The content of the file that will be attached
+    :type file_content: string
+    :param file_suffix: File suffix. This should be used to set the file
+        extension for applications that are sensitive to this.
+    :type file_suffix: string
+    :returns: None
+    """
+    with tempfile.NamedTemporaryFile(mode='w', suffix=file_suffix) as fp:
+        fp.write(file_content)
+        fp.flush()
+        model.attach_resource(
+            application_name, resource_name, fp.name)

zaza/openstack/utilities/openstack.py

@@ -16,9 +16,31 @@
 
 This module contains a number of functions for interacting with OpenStack.
 """
+import collections
+import copy
+import datetime
+import enum
+import io
+import itertools
+import juju_wait
+import logging
+import os
+import paramiko
+import re
+import shutil
+import six
+import subprocess
+import sys
+import tempfile
+import tenacity
+import textwrap
+import urllib
+
+
 from .os_versions import (
     OPENSTACK_CODENAMES,
     SWIFT_CODENAMES,
+    OVN_CODENAMES,
     PACKAGE_CODENAMES,
     OPENSTACK_RELEASES_PAIRS,
 )
@@ -41,27 +63,15 @@ from keystoneauth1.identity import (
 import zaza.openstack.utilities.cert as cert
 import zaza.utilities.deployment_env as deployment_env
 import zaza.utilities.juju as juju_utils
+import zaza.utilities.maas
 from novaclient import client as novaclient_client
 from neutronclient.v2_0 import client as neutronclient
 from neutronclient.common import exceptions as neutronexceptions
 from octaviaclient.api.v2 import octavia as octaviaclient
 from swiftclient import client as swiftclient
+from manilaclient import client as manilaclient
 
-import datetime
-import io
-import itertools
-import juju_wait
-import logging
-import os
-import paramiko
-import re
-import six
-import subprocess
-import sys
-import tempfile
-import tenacity
-import textwrap
-import urllib
+from juju.errors import JujuError
 
 import zaza
 
@@ -123,6 +133,10 @@ CHARM_TYPES = {
         'pkg': 'ceph-common',
         'origin_setting': 'source'
     },
+    'placement': {
+        'pkg': 'placement-common',
+        'origin_setting': 'openstack-origin'
+    },
 }
 
 # Older tests use the order the services appear in the list to imply
@@ -143,6 +157,7 @@ UPGRADE_SERVICES = [
      'type': CHARM_TYPES['openstack-dashboard']},
     {'name': 'ovn-central', 'type': CHARM_TYPES['ovn-central']},
     {'name': 'ceph-mon', 'type': CHARM_TYPES['ceph-mon']},
+    {'name': 'placement', 'type': CHARM_TYPES['placement']},
 ]
 
 
@@ -169,20 +184,83 @@ WORKLOAD_STATUS_EXCEPTIONS = {
              'ceilometer and gnocchi')}}
 
 # For vault TLS certificates
+CACERT_FILENAME_FORMAT = "{}_juju_ca_cert.crt"
+CERT_PROVIDERS = ['vault']
+REMOTE_CERT_DIR = "/usr/local/share/ca-certificates"
 KEYSTONE_CACERT = "keystone_juju_ca_cert.crt"
 KEYSTONE_REMOTE_CACERT = (
     "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT))
-KEYSTONE_LOCAL_CACERT = ("tests/{}".format(KEYSTONE_CACERT))
+
+
+async def async_block_until_ca_exists(application_name, ca_cert,
+                                      model_name=None, timeout=2700):
+    """Block until a CA cert is on all units of application_name.
+
+    :param application_name: Name of application to check
+    :type application_name: str
+    :param ca_cert: The certificate content.
+    :type ca_cert: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :param timeout: How long in seconds to wait
+    :type timeout: int
+    """
+    async def _check_ca_present(model, ca_files):
+        units = model.applications[application_name].units
+        for ca_file in ca_files:
+            for unit in units:
+                try:
+                    output = await unit.run('cat {}'.format(ca_file))
+                    contents = output.data.get('results').get('Stdout', '')
+                    if ca_cert not in contents:
+                        break
+                # libjuju throws a generic error for connection failure. So we
+                # cannot differentiate between a connectivity issue and a
+                # target file not existing error. For now just assume the
+                # latter.
+                except JujuError:
+                    break
+            else:
+                # The CA was found in `ca_file` on all units.
+                return True
+        else:
+            return False
+    ca_files = await _async_get_remote_ca_cert_file_candidates(
+        application_name,
+        model_name=model_name)
+    async with zaza.model.run_in_model(model_name) as model:
+        await zaza.model.async_block_until(
+            lambda: _check_ca_present(model, ca_files), timeout=timeout)
+
+block_until_ca_exists = zaza.model.sync_wrapper(async_block_until_ca_exists)
+
+
+def get_cacert_absolute_path(filename):
+    """Build string containing location of the CA Certificate file.
+
+    :param filename: Expected filename for CA Certificate file.
+    :type filename: str
+    :returns: Absolute path to file containing CA Certificate
+    :rtype: str
+    """
+    return os.path.join(
+        deployment_env.get_tmpdir(), filename)
 
 
 def get_cacert():
     """Return path to CA Certificate bundle for verification during test.
 
     :returns: Path to CA Certificate bundle or None.
-    :rtype: Optional[str]
+    :rtype: Union[str, None]
     """
-    if os.path.exists(KEYSTONE_LOCAL_CACERT):
-        return KEYSTONE_LOCAL_CACERT
+    for _provider in CERT_PROVIDERS:
+        _cert = get_cacert_absolute_path(
+            CACERT_FILENAME_FORMAT.format(_provider))
+        if os.path.exists(_cert):
+            return _cert
+    _keystone_local_cacert = get_cacert_absolute_path(KEYSTONE_CACERT)
+    if os.path.exists(_keystone_local_cacert):
+        return _keystone_local_cacert
 
 
 # OpenStack Client helpers
@@ -248,15 +326,17 @@ def get_designate_session_client(**kwargs):
                            **kwargs)
 
 
-def get_nova_session_client(session):
+def get_nova_session_client(session, version=2):
     """Return novaclient authenticated by keystone session.
 
     :param session: Keystone session object
     :type session: keystoneauth1.session.Session object
+    :param version: Version of client to request.
+    :type version: float
     :returns: Authenticated novaclient
     :rtype: novaclient.Client object
     """
-    return novaclient_client.Client(2, session=session)
+    return novaclient_client.Client(version, session=session)
 
 
 def get_neutron_session_client(session):
@@ -370,6 +450,19 @@ def get_aodh_session_client(session):
     return aodh_client.Client(session=session)
 
 
+def get_manila_session_client(session, version='2'):
+    """Return Manila client authenticated by keystone session.
+
+    :param session: Keystone session object
+    :type session: keystoneauth1.session.Session object
+    :param version: Manila API version
+    :type version: str
+    :returns: Authenticated manilaclient
+    :rtype: manilaclient.Client
+    """
+    return manilaclient.Client(session=session, client_version=version)
+
+
 def get_keystone_scope(model_name=None):
     """Return Keystone scope based on OpenStack release of the overcloud.
 
@@ -711,6 +804,211 @@ def add_interface_to_netplan(server_name, mac_address):
             model.run_on_unit(unit_name, "sudo netplan apply")
 
 
+class OpenStackNetworkingTopology(enum.Enum):
+    """OpenStack Charms Network Topologies."""
+
+    ML2_OVS = 'ML2+OVS'
+    ML2_OVS_DVR = 'ML2+OVS+DVR'
+    ML2_OVS_DVR_SNAT = 'ML2+OVS+DVR, no dedicated GWs'
+    ML2_OVN = 'ML2+OVN'
+
+
+CharmedOpenStackNetworkingData = collections.namedtuple(
+    'CharmedOpenStackNetworkingData',
+    [
+        'topology',
+        'application_names',
+        'unit_machine_ids',
+        'port_config_key',
+        'other_config',
+    ])
+
+
+def get_charm_networking_data(limit_gws=None):
+    """Inspect Juju model, determine networking topology and return data.
+
+    :param limit_gws: Limit the number of gateways that get a port attached
+    :type limit_gws: Optional[int]
+    :rtype: CharmedOpenStackNetworkingData[
+                OpenStackNetworkingTopology,
+                List[str],
+                Iterator[str],
+                str,
+                Dict[str,str]]
+    :returns: Named Tuple with networking data, example:
+        CharmedOpenStackNetworkingData(
+            OpenStackNetworkingTopology.ML2_OVN,
+            ['ovn-chassis', 'ovn-dedicated-chassis'],
+            ['machine-id-1', 'machine-id-2'],         # generator object
+            'bridge-interface-mappings',
+            {'ovn-bridge-mappings': 'physnet1:br-ex'})
+    :raises: RuntimeError
+    """
+    # Initialize defaults, these will be amended to fit the reality of the
+    # model in the checks below.
+    topology = OpenStackNetworkingTopology.ML2_OVS
+    other_config = {}
+    port_config_key = (
+        'data-port' if not deprecated_external_networking() else 'ext-port')
+    unit_machine_ids = []
+    application_names = []
+
+    if dvr_enabled():
+        if ngw_present():
+            application_names = ['neutron-gateway', 'neutron-openvswitch']
+            topology = OpenStackNetworkingTopology.ML2_OVS_DVR
+        else:
+            application_names = ['neutron-openvswitch']
+            topology = OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT
+        unit_machine_ids = itertools.islice(
+            itertools.chain(
+                get_ovs_uuids(),
+                get_gateway_uuids()),
+            limit_gws)
+    elif ngw_present():
+        unit_machine_ids = itertools.islice(
+            get_gateway_uuids(), limit_gws)
+        application_names = ['neutron-gateway']
+    elif ovn_present():
+        topology = OpenStackNetworkingTopology.ML2_OVN
+        unit_machine_ids = itertools.islice(get_ovn_uuids(), limit_gws)
+        application_names = ['ovn-chassis']
+        try:
+            ovn_dc_name = 'ovn-dedicated-chassis'
+            model.get_application(ovn_dc_name)
+            application_names.append(ovn_dc_name)
+        except KeyError:
+            # ovn-dedicated-chassis not in deployment
+            pass
+        port_config_key = 'bridge-interface-mappings'
+        other_config.update({'ovn-bridge-mappings': 'physnet1:br-ex'})
+    else:
+        raise RuntimeError('Unable to determine charm network topology.')
+
+    return CharmedOpenStackNetworkingData(
+        topology,
+        application_names,
+        unit_machine_ids,
+        port_config_key,
+        other_config)
+
+
+def create_additional_port_for_machines(novaclient, neutronclient, net_id,
+                                        unit_machine_ids,
+                                        add_dataport_to_netplan=False):
+    """Create additional port for machines for use with external networking.
+
+    :param novaclient: Undercloud Authenticated novaclient.
+    :type novaclient: novaclient.Client object
+    :param neutronclient: Undercloud Authenticated neutronclient.
+    :type neutronclient: neutronclient.Client object
+    :param net_id: Network ID to create ports on.
+    :type net_id: string
+    :param unit_machine_ids: Juju provider specific machine IDs for which we
+                             should add ports on.
+    :type unit_machine_ids: Iterator[str]
+    :param add_dataport_to_netplan: Whether the newly created port should be
+                                    added to instance system configuration so
+                                    that it is brought up on instance reboot.
+    :type add_dataport_to_netplan: Optional[bool]
+    :returns: List of MAC addresses for created ports.
+    :rtype: List[str]
+    :raises: RuntimeError
+    """
+    eligible_machines = 0
+    for uuid in unit_machine_ids:
+        eligible_machines += 1
+        server = novaclient.servers.get(uuid)
+        ext_port_name = "{}_ext-port".format(server.name)
+        for port in neutronclient.list_ports(device_id=server.id)['ports']:
+            if port['name'] == ext_port_name:
+                logging.warning(
+                    'Instance {} already has additional port, skipping.'
+                    .format(server.id))
+                break
+        else:
+            logging.info('Attaching additional port to instance ("{}"), '
+                         'connected to net id: {}'
+                         .format(uuid, net_id))
+            body_value = {
+                "port": {
+                    "admin_state_up": True,
+                    "name": ext_port_name,
+                    "network_id": net_id,
+                    "port_security_enabled": False,
+                }
+            }
+            port = neutronclient.create_port(body=body_value)
+            server.interface_attach(port_id=port['port']['id'],
+                                    net_id=None, fixed_ip=None)
+            if add_dataport_to_netplan:
+                mac_address = get_mac_from_port(port, neutronclient)
+                add_interface_to_netplan(server.name,
+                                         mac_address=mac_address)
+    if not eligible_machines:
+        # NOTE: unit_machine_ids may be an iterator so testing it for contents
+        # or length prior to iterating over it is futile.
+        raise RuntimeError('Unable to determine UUIDs for machines to attach '
+                           'external networking to.')
+
+    # Retrieve the just created ports from Neutron so that we can provide our
+    # caller with their MAC addresses.
+    return [
+        port['mac_address']
+        for port in neutronclient.list_ports(network_id=net_id)['ports']
+        if 'ext-port' in port['name']
+    ]
+
+
+def configure_networking_charms(networking_data, macs, use_juju_wait=True):
+    """Configure external networking for networking charms.
+
+    :param networking_data: Data on networking charm topology.
+    :type networking_data: CharmedOpenStackNetworkingData
+    :param macs: MAC addresses of ports for use with external networking.
+    :type macs: Iterator[str]
+    :param use_juju_wait: Whether to use juju wait to wait for the model to
+        settle once the gateway has been configured. Default is True
+    :type use_juju_wait: Optional[bool]
+    """
+    br_mac_fmt = 'br-ex:{}' if not deprecated_external_networking() else '{}'
+    br_mac = [
+        br_mac_fmt.format(mac)
+        for mac in macs
+    ]
+
+    config = copy.deepcopy(networking_data.other_config)
+    config.update({networking_data.port_config_key: ' '.join(sorted(br_mac))})
+
+    for application_name in networking_data.application_names:
+        logging.info('Setting {} on {}'.format(
+            config, application_name))
+        current_data_port = get_application_config_option(
+            application_name,
+            networking_data.port_config_key)
+        if current_data_port == config[networking_data.port_config_key]:
+            logging.info('Config already set to value')
+            return
+
+        model.set_application_config(
+            application_name,
+            configuration=config)
+    # NOTE(fnordahl): We are stuck with juju_wait until we figure out how
+    # to deal with all the non ['active', 'idle', 'Unit is ready.']
+    # workload/agent states and msgs that our mojo specs are exposed to.
+    if use_juju_wait:
+        juju_wait.wait(wait_for_workload=True)
+    else:
+        zaza.model.wait_for_agent_status()
+        # TODO: shouldn't access get_charm_config() here as it relies on
+        # ./tests/tests.yaml existing by default (regardless of the
+        # fatal=False) ... it's not great design.
+        test_config = zaza.charm_lifecycle.utils.get_charm_config(
+            fatal=False)
+        zaza.model.wait_for_application_states(
+            states=test_config.get('target_deploy_status', {}))
+
+
 def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
                                add_dataport_to_netplan=False,
                                limit_gws=None,
@@ -729,123 +1027,46 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None,
         settle once the gateway has been configured. Default is True
     :type use_juju_wait: boolean
     """
-    deprecated_extnet_mode = deprecated_external_networking()
-
-    port_config_key = 'data-port'
-    if deprecated_extnet_mode:
-        port_config_key = 'ext-port'
-
-    config = {}
-    if dvr_enabled():
-        uuids = itertools.islice(itertools.chain(get_ovs_uuids(),
-                                                 get_gateway_uuids()),
-                                 limit_gws)
+    networking_data = get_charm_networking_data(limit_gws=limit_gws)
+    if networking_data.topology in (
+            OpenStackNetworkingTopology.ML2_OVS_DVR,
+            OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT):
         # If dvr, do not attempt to persist nic in netplan
         # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78
         add_dataport_to_netplan = False
-        application_names = ['neutron-openvswitch']
-        try:
-            ngw = 'neutron-gateway'
-            model.get_application(ngw)
-            application_names.append(ngw)
-        except KeyError:
-            # neutron-gateway not in deployment
-            pass
-    elif ngw_present():
-        uuids = itertools.islice(get_gateway_uuids(), limit_gws)
-        application_names = ['neutron-gateway']
-    elif ovn_present():
-        uuids = itertools.islice(get_ovn_uuids(), limit_gws)
-        application_names = ['ovn-chassis']
-        try:
-            ovn_dc_name = 'ovn-dedicated-chassis'
-            model.get_application(ovn_dc_name)
-            application_names.append(ovn_dc_name)
-        except KeyError:
-            # ovn-dedicated-chassis not in deployment
-            pass
-        port_config_key = 'bridge-interface-mappings'
-        config.update({'ovn-bridge-mappings': 'physnet1:br-ex'})
-        add_dataport_to_netplan = True
-    else:
-        raise RuntimeError('Unable to determine charm network topology.')
 
     if not net_id:
         net_id = get_admin_net(neutronclient)['id']
 
-    ports_created = 0
-    for uuid in uuids:
-        server = novaclient.servers.get(uuid)
-        ext_port_name = "{}_ext-port".format(server.name)
-        for port in neutronclient.list_ports(device_id=server.id)['ports']:
-            if port['name'] == ext_port_name:
-                logging.warning(
-                    'Neutron Gateway already has additional port')
-                break
-        else:
-            logging.info('Attaching additional port to instance ("{}"), '
-                         'connected to net id: {}'
-                         .format(uuid, net_id))
-            body_value = {
-                "port": {
-                    "admin_state_up": True,
-                    "name": ext_port_name,
-                    "network_id": net_id,
-                    "port_security_enabled": False,
-                }
-            }
-            port = neutronclient.create_port(body=body_value)
-            ports_created += 1
-            server.interface_attach(port_id=port['port']['id'],
-                                    net_id=None, fixed_ip=None)
-            if add_dataport_to_netplan:
-                mac_address = get_mac_from_port(port, neutronclient)
-                add_interface_to_netplan(server.name,
-                                         mac_address=mac_address)
-    if not ports_created:
-        # NOTE: uuids is an iterator so testing it for contents or length prior
-        # to iterating over it is futile.
-        raise RuntimeError('Unable to determine UUIDs for machines to attach '
-                           'external networking to.')
+    macs = create_additional_port_for_machines(
+        novaclient, neutronclient, net_id, networking_data.unit_machine_ids,
+        add_dataport_to_netplan)
 
-    ext_br_macs = []
-    for port in neutronclient.list_ports(network_id=net_id)['ports']:
-        if 'ext-port' in port['name']:
-            if deprecated_extnet_mode:
-                ext_br_macs.append(port['mac_address'])
-            else:
-                ext_br_macs.append('br-ex:{}'.format(port['mac_address']))
-    ext_br_macs.sort()
-    ext_br_macs_str = ' '.join(ext_br_macs)
+    if macs:
+        configure_networking_charms(
+            networking_data, macs, use_juju_wait=use_juju_wait)
 
-    if ext_br_macs:
-        config.update({port_config_key: ext_br_macs_str})
-        for application_name in application_names:
-            logging.info('Setting {} on {}'.format(
-                config, application_name))
-            current_data_port = get_application_config_option(application_name,
-                                                              port_config_key)
-            if current_data_port == ext_br_macs_str:
-                logging.info('Config already set to value')
-                return
 
-            model.set_application_config(
-                application_name,
-                configuration=config)
-        # NOTE(fnordahl): We are stuck with juju_wait until we figure out how
-        # to deal with all the non ['active', 'idle', 'Unit is ready.']
-        # workload/agent states and msgs that our mojo specs are exposed to.
-        if use_juju_wait:
-            juju_wait.wait(wait_for_workload=True)
-        else:
-            zaza.model.wait_for_agent_status()
-            # TODO: shouldn't access get_charm_config() here as it relies on
-            # ./tests/tests.yaml existing by default (regardless of the
-            # fatal=False) ... it's not great design.
-            test_config = zaza.charm_lifecycle.utils.get_charm_config(
-                fatal=False)
-            zaza.model.wait_for_application_states(
-                states=test_config.get('target_deploy_status', {}))
+def configure_charmed_openstack_on_maas(network_config, limit_gws=None):
+    """Configure networking charms for charm-based OVS config on MAAS provider.
+
+    :param network_config: Network configuration as provided in environment.
+    :type network_config: Dict[str]
+    :param limit_gws: Limit the number of gateways that get a port attached
+    :type limit_gws: Optional[int]
+    """
+    networking_data = get_charm_networking_data(limit_gws=limit_gws)
+    macs = [
+        mim.mac
+        for mim in zaza.utilities.maas.get_macs_from_cidr(
+            zaza.utilities.maas.get_maas_client_from_juju_cloud_data(
+                zaza.model.get_cloud_data()),
+            network_config['external_net_cidr'],
+            link_mode=zaza.utilities.maas.LinkMode.LINK_UP)
+    ]
+    if macs:
+        configure_networking_charms(
+            networking_data, macs, use_juju_wait=False)
 
 
 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
@@ -1474,8 +1695,23 @@ def get_swift_codename(version):
     :returns: Codename for swift
     :rtype: string
     """
-    codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v]
-    return codenames[0]
+    return _get_special_codename(version, SWIFT_CODENAMES)
+
+
+def get_ovn_codename(version):
+    """Determine OpenStack codename that corresponds to OVN version.
+
+    :param version: Version of OVN
+    :type version: string
+    :returns: Codename for OVN
+    :rtype: string
+    """
+    return _get_special_codename(version, OVN_CODENAMES)
+
+
+def _get_special_codename(version, codenames):
+    found = [k for k, v in six.iteritems(codenames) if version in v]
+    return found[0]
 
 
 def get_os_code_info(package, pkg_version):
@@ -1488,7 +1724,6 @@ def get_os_code_info(package, pkg_version):
     :returns: Codename for package
     :rtype: string
     """
-    # {'code_num': entry, 'code_name': OPENSTACK_CODENAMES[entry]}
     # Remove epoch if it exists
     if ':' in pkg_version:
         pkg_version = pkg_version.split(':')[1:][0]
@@ -1512,13 +1747,55 @@ def get_os_code_info(package, pkg_version):
         # < Liberty co-ordinated project versions
         if 'swift' in package:
             return get_swift_codename(vers)
+        elif 'ovn' in package:
+            return get_ovn_codename(vers)
         else:
             return OPENSTACK_CODENAMES[vers]
 
 
+def get_openstack_release(application, model_name=None):
+    """Return the openstack release codename based on /etc/openstack-release.
+
+    This will only return a codename if the openstack-release package is
+    installed on the unit.
+
+    :param application: Application name
+    :type application: string
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :returns: OpenStack release codename for application
+    :rtype: string
+    """
+    versions = []
+    units = model.get_units(application, model_name=model_name)
+    for unit in units:
+        cmd = 'cat /etc/openstack-release | grep OPENSTACK_CODENAME'
+        try:
+            out = juju_utils.remote_run(unit.entity_id, cmd,
+                                        model_name=model_name)
+        except model.CommandRunFailed:
+            logging.debug('Fall back to version check for OpenStack codename')
+        else:
+            codename = out.split('=')[1].strip()
+            versions.append(codename)
+    if len(set(versions)) == 0:
+        return None
+    elif len(set(versions)) > 1:
+        raise Exception('Unexpected mix of OpenStack releases for {}: {}',
+                        application, versions)
+    return versions[0]
+
+
 def get_current_os_versions(deployed_applications, model_name=None):
     """Determine OpenStack codename of deployed applications.
 
+    Initially, see if the openstack-release pkg is available and use it
+    instead.
+
+    If it isn't then it falls back to the existing method of checking the
+    version of the package passed and then resolving the version from that
+    using lookup tables.
+
     :param deployed_applications: List of deployed applications
     :type deployed_applications: list
     :param model_name: Name of model to query.
@@ -1530,12 +1807,18 @@ def get_current_os_versions(deployed_applications, model_name=None):
     for application in UPGRADE_SERVICES:
         if application['name'] not in deployed_applications:
             continue
+        logging.info("looking at application: {}".format(application))
 
-        version = generic_utils.get_pkg_version(application['name'],
-                                                application['type']['pkg'],
-                                                model_name=model_name)
-        versions[application['name']] = (
-            get_os_code_info(application['type']['pkg'], version))
+        codename = get_openstack_release(application['name'],
+                                         model_name=model_name)
+        if codename:
+            versions[application['name']] = codename
+        else:
+            version = generic_utils.get_pkg_version(application['name'],
+                                                    application['type']['pkg'],
+                                                    model_name=model_name)
+            versions[application['name']] = (
+                get_os_code_info(application['type']['pkg'], version))
     return versions
 
 
@@ -1578,15 +1861,19 @@ def get_current_os_release_pair(application='keystone'):
     return '{}_{}'.format(series, os_version)
 
 
-def get_os_release(release_pair=None):
+def get_os_release(release_pair=None, application='keystone'):
     """Return index of release in OPENSTACK_RELEASES_PAIRS.
 
+    :param release_pair: OpenStack release pair eg 'focal_ussuri'
+    :type release_pair: string
+    :param application: Name of application to derive release pair from.
+    :type application: string
     :returns: Index of the release
     :rtype: int
     :raises: exceptions.ReleasePairNotFound
     """
     if release_pair is None:
-        release_pair = get_current_os_release_pair()
+        release_pair = get_current_os_release_pair(application=application)
     try:
         index = OPENSTACK_RELEASES_PAIRS.index(release_pair)
     except ValueError:
@@ -1787,29 +2074,82 @@ def get_overcloud_auth(address=None, model_name=None):
             'OS_PROJECT_DOMAIN_NAME': 'admin_domain',
             'API_VERSION': 3,
         }
-    if tls_rid:
-        unit = model.get_first_unit_name('keystone', model_name=model_name)
-
-        # ensure that the path to put the local cacert in actually exists.  The
-        # assumption that 'tests/' exists for, say, mojo is false.
-        # Needed due to:
-        # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2
-        _dir = os.path.dirname(KEYSTONE_LOCAL_CACERT)
-        if not os.path.exists(_dir):
-            os.makedirs(_dir)
-
-        model.scp_from_unit(
-            unit,
-            KEYSTONE_REMOTE_CACERT,
-            KEYSTONE_LOCAL_CACERT)
-
-        if os.path.exists(KEYSTONE_LOCAL_CACERT):
-            os.chmod(KEYSTONE_LOCAL_CACERT, 0o644)
-            auth_settings['OS_CACERT'] = KEYSTONE_LOCAL_CACERT
+    local_ca_cert = get_remote_ca_cert_file('keystone', model_name=model_name)
+    if local_ca_cert:
+        auth_settings['OS_CACERT'] = local_ca_cert
 
     return auth_settings
 
 
+async def _async_get_remote_ca_cert_file_candidates(application,
+                                                    model_name=None):
+    """Return a list of possible remote CA file names.
+
+    :param application: Name of application to examine.
+    :type application: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :returns: List of paths to possible ca files.
+    :rtype: List[str]
+    """
+    cert_files = []
+    for _provider in CERT_PROVIDERS:
+        tls_rid = await model.async_get_relation_id(
+            application,
+            _provider,
+            model_name=model_name,
+            remote_interface_name='certificates')
+        if tls_rid:
+            cert_files.append(
+                REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format(
+                    _provider))
+    cert_files.append(KEYSTONE_REMOTE_CACERT)
+    return cert_files
+
+_get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper(
+    _async_get_remote_ca_cert_file_candidates)
+
+
+def get_remote_ca_cert_file(application, model_name=None):
+    """Collect CA certificate from application.
+
+    :param application: Name of application to collect file from.
+    :type application: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :returns: Path to cafile
+    :rtype: str
+    """
+    unit = model.get_first_unit_name(application, model_name=model_name)
+    local_cert_file = None
+    cert_files = _get_remote_ca_cert_file_candidates(
+        application,
+        model_name=model_name)
+    for cert_file in cert_files:
+        _local_cert_file = get_cacert_absolute_path(
+            os.path.basename(cert_file))
+        with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca:
+            try:
+                model.scp_from_unit(
+                    unit,
+                    cert_file,
+                    _tmp_ca.name)
+            except JujuError:
+                continue
+            # ensure that the path to put the local cacert in actually exists.
+            # The assumption that 'tests/' exists for, say, mojo is false.
+            # Needed due to:
+            # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2
+            _dir = os.path.dirname(_local_cert_file)
+            if not os.path.exists(_dir):
+                os.makedirs(_dir)
+            shutil.move(_tmp_ca.name, _local_cert_file)
+            os.chmod(_local_cert_file, 0o644)
+            local_cert_file = _local_cert_file
+            break
+    return local_cert_file
+
+
 def get_urllib_opener():
     """Create a urllib opener taking into account proxy settings.
 
@@ -1881,7 +2221,8 @@ def download_image(image_url, target_file):
 
 def _resource_reaches_status(resource, resource_id,
                              expected_status='available',
-                             msg='resource'):
+                             msg='resource',
+                             resource_attribute='status'):
     """Wait for an openstack resources status to reach an expected status.
 
        Wait for an openstack resources status to reach an expected status
@@ -1896,20 +2237,22 @@ def _resource_reaches_status(resource, resource_id,
     :param expected_status: status to expect resource to reach
     :type expected_status: str
     :param msg: text to identify purpose in logging
-    :type msy: str
+    :type msg: str
+    :param resource_attribute: Resource attribute to check against
+    :type resource_attribute: str
     :raises: AssertionError
     """
-    resource_status = resource.get(resource_id).status
-    logging.info(resource_status)
-    assert resource_status == expected_status, (
-        "Resource in {} state, waiting for {}" .format(resource_status,
-                                                       expected_status,))
+    resource_status = getattr(resource.get(resource_id), resource_attribute)
+    logging.info("{}: resource {} in {} state, waiting for {}".format(
+        msg, resource_id, resource_status, expected_status))
+    assert resource_status == expected_status
 
 
 def resource_reaches_status(resource,
                             resource_id,
                             expected_status='available',
                             msg='resource',
+                            resource_attribute='status',
                             wait_exponential_multiplier=1,
                             wait_iteration_max_time=60,
                             stop_after_attempt=8,
@@ -1929,6 +2272,8 @@ def resource_reaches_status(resource,
     :type expected_status: str
     :param msg: text to identify purpose in logging
     :type msg: str
+    :param resource_attribute: Resource attribute to check against
+    :type resource_attribute: str
     :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier
                                         seconds between each retry
     :type wait_exponential_multiplier: int
@@ -1950,7 +2295,8 @@ def resource_reaches_status(resource,
         resource,
         resource_id,
         expected_status,
-        msg)
+        msg,
+        resource_attribute)
 
 
 def _resource_removed(resource, resource_id, msg="resource"):
@@ -1965,8 +2311,8 @@ def _resource_removed(resource, resource_id, msg="resource"):
     :raises: AssertionError
     """
     matching = [r for r in resource.list() if r.id == resource_id]
-    logging.debug("Resource {} still present".format(resource_id))
-    assert len(matching) == 0, "Resource {} still present".format(resource_id)
+    logging.debug("{}: resource {} still present".format(msg, resource_id))
+    assert len(matching) == 0
 
 
 def resource_removed(resource,
@@ -2060,7 +2406,8 @@ def delete_volume_backup(cinder, vol_backup_id):
 
 
 def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
-                           visibility='public', container_format='bare'):
+                           visibility='public', container_format='bare',
+                           backend=None, force_import=False):
     """Upload the given image to glance and apply the given label.
 
     :param glance: Authenticated glanceclient
@@ -2077,6 +2424,9 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
                              format that also contains metadata about the
                              actual virtual machine.
     :type container_format: str
+    :param force_import: Force the use of glance image import
+        instead of direct upload
+    :type force_import: boolean
     :returns: glance image pointer
     :rtype: glanceclient.common.utils.RequestIdProxy
     """
@@ -2086,7 +2436,15 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
         disk_format=disk_format,
         visibility=visibility,
         container_format=container_format)
-    glance.images.upload(image.id, open(local_path, 'rb'))
+
+    if force_import:
+        logging.info('Forcing image import')
+        glance.images.stage(image.id, open(local_path, 'rb'))
+        glance.images.image_import(
+            image.id, method='glance-direct', backend=backend)
+    else:
+        glance.images.upload(
+            image.id, open(local_path, 'rb'), backend=backend)
 
     resource_reaches_status(
         glance.images,
@@ -2098,7 +2456,9 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
 
 
 def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
-                 properties=None):
+                 properties=None, backend=None, disk_format='qcow2',
+                 visibility='public', container_format='bare',
+                 force_import=False):
     """Download the image and upload it to glance.
 
     Download an image from image_url and upload it to glance labelling
@@ -2117,6 +2477,9 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
     :type tags: list of str
     :param properties: Properties and values to add to image
     :type properties: dict
+    :param force_import: Force the use of glance image import
+        instead of direct upload
+    :type force_import: boolean
     :returns: glance image pointer
     :rtype: glanceclient.common.utils.RequestIdProxy
     """
@@ -2130,9 +2493,16 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
     local_path = os.path.join(image_cache_dir, img_name)
 
     if not os.path.exists(local_path):
+        logging.info('Downloading {} ...'.format(image_url))
         download_image(image_url, local_path)
+    else:
+        logging.info('Cached image found at {} - Skipping download'.format(
+            local_path))
 
-    image = upload_image_to_glance(glance, local_path, image_name)
+    image = upload_image_to_glance(
+        glance, local_path, image_name, backend=backend,
+        disk_format=disk_format, visibility=visibility,
+        container_format=container_format, force_import=force_import)
     for tag in tags:
         result = glance.image_tags.update(image.id, tag)
         logging.debug(
@@ -2197,6 +2567,40 @@ def attach_volume(nova, volume_id, instance_id):
                                              device='/dev/vdx')
 
 
+def failover_cinder_volume_host(cinder, backend_name='cinder-ceph',
+                                target_backend_id='ceph',
+                                target_status='disabled',
+                                target_replication_status='failed-over'):
+    """Failover Cinder volume host with replication enabled.
+
+    :param cinder: Authenticated cinderclient
+    :type cinder: cinder.Client
+    :param backend_name: Cinder volume backend name with
+                         replication enabled.
+    :type backend_name: str
+    :param target_backend_id: Failover target Cinder backend id.
+    :type target_backend_id: str
+    :param target_status: Target Cinder volume status after failover.
+    :type target_status: str
+    :param target_replication_status: Target Cinder volume replication
+                                      status after failover.
+    :type target_replication_status: str
+    :raises: AssertionError
+    """
+    host = 'cinder@{}'.format(backend_name)
+    logging.info('Failover Cinder volume host %s to backend_id %s',
+                 host, target_backend_id)
+    cinder.services.failover_host(host=host, backend_id=target_backend_id)
+    for attempt in tenacity.Retrying(
+            retry=tenacity.retry_if_exception_type(AssertionError),
+            stop=tenacity.stop_after_attempt(10),
+            wait=tenacity.wait_exponential(multiplier=1, min=2, max=10)):
+        with attempt:
+            svc = cinder.services.list(host=host, binary='cinder-volume')[0]
+            assert svc.status == target_status
+            assert svc.replication_status == target_replication_status
+
+
 def create_volume_backup(cinder, volume_id, name=None):
     """Create cinder volume backup.
 

zaza/openstack/utilities/openstack_upgrade.py

@@ -95,8 +95,8 @@ async def async_action_unit_upgrade(units, model_name=None):
 action_unit_upgrade = sync_wrapper(async_action_unit_upgrade)
 
 
-def action_upgrade_group(applications, model_name=None):
-    """Upgrade units using action managed upgrades.
+def action_upgrade_apps(applications, model_name=None):
+    """Upgrade units in the applications using action managed upgrades.
 
     Upgrade all units of the given applications using action managed upgrades.
     This involves the following process:
@@ -142,6 +142,45 @@ def action_upgrade_group(applications, model_name=None):
 
         done.extend(target)
 
+    # Ensure that mysql-innodb-cluster has at least one R/W group (it can get
+    # into a state where all are R/O whilst it is sorting itself out after an
+    # openstack_upgrade
+    if "mysql-innodb-cluster" in applications:
+        block_until_mysql_innodb_cluster_has_rw(model_name)
+
+    # Now we need to wait for the model to go back to idle.
+    zaza.model.block_until_all_units_idle(model_name)
+
+
+async def async_block_until_mysql_innodb_cluster_has_rw(model=None,
+                                                        timeout=None):
+    """Block until the mysql-innodb-cluster is in a healthy state.
+
+    Curiously, after a series of pauses and restarts (e.g. during an upgrade)
+    the mysql-innodb-cluster charms may not yet have agreed which one is the
+    R/W node; i.e. they are all R/O.  Anyway, eventually they sort it out and
+    one jumps to the front and says "it's me!".  This is detected, externally,
+    by the status line including R/W in the output.
+
+    This function blocks until that happens so that no charm attempts to have a
+    chat with the mysql server before it has settled, thus breaking the whole
+    test.
+    """
+    async def async_check_workload_messages_for_rw(model=None):
+        """Return True if a least one work message contains R/W."""
+        status = await zaza.model.async_get_status()
+        app_status = status.applications.get("mysql-innodb-cluster")
+        units_data = app_status.units.values()
+        workload_statuses = [d.workload_status.info for d in units_data]
+        return any("R/W" in s for s in workload_statuses)
+
+    await zaza.model.async_block_until(async_check_workload_messages_for_rw,
+                                       timeout=timeout)
+
+
+block_until_mysql_innodb_cluster_has_rw = sync_wrapper(
+    async_block_until_mysql_innodb_cluster_has_rw)
+
 
 def set_upgrade_application_config(applications, new_source,
                                    action_managed=True, model_name=None):
@@ -150,7 +189,7 @@ def set_upgrade_application_config(applications, new_source,
     Set the charm config for upgrade.
 
     :param applications: List of application names.
-    :type applications: []
+    :type applications: List[str]
     :param new_source: New package origin.
     :type new_source: str
     :param action_managed: Whether to set action-managed-upgrade config option.
@@ -180,8 +219,8 @@ def set_upgrade_application_config(applications, new_source,
 def is_action_upgradable(app, model_name=None):
     """Can application be upgraded using action managed upgrade method.
 
-    :param new_source: New package origin.
-    :type new_source: str
+    :param app: The application to check
+    :type app: str
     :param model_name: Name of model to query.
     :type model_name: str
     :returns: Whether app be upgraded using action managed upgrade method.
@@ -196,66 +235,95 @@ def is_action_upgradable(app, model_name=None):
     return supported
 
 
-def run_action_upgrade(group, new_source, model_name=None):
+def is_already_upgraded(app, new_src, model_name=None):
+    """Return True if the app has already been upgraded.
+
+    :param app: The application to check
+    :type app: str
+    :param new_src: the new source (distro, cloud:x-y, etc.)
+    :type new_src: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :returns: Whether app be upgraded using action managed upgrade method.
+    :rtype: bool
+    """
+    config = zaza.model.get_application_config(app, model_name=model_name)
+    try:
+        src = config['openstack-origin']['value']
+        key_was = 'openstack-origin'
+    except KeyError:
+        src = config['source']['value']
+        key_was = 'source'
+    logging.info("origin for {} is {}={}".format(app, key_was, src))
+    return src == new_src
+
+
+def run_action_upgrades(apps, new_source, model_name=None):
     """Upgrade payload of all applications in group using action upgrades.
 
-    :param group: List of applications to upgrade.
-    :type group
+    :param apps: List of applications to upgrade.
+    :type apps: List[str]
     :param new_source: New package origin.
     :type new_source: str
     :param model_name: Name of model to query.
     :type model_name: str
     """
-    set_upgrade_application_config(group, new_source, model_name=model_name)
-    action_upgrade_group(group, model_name=model_name)
+    set_upgrade_application_config(apps, new_source, model_name=model_name)
+    action_upgrade_apps(apps, model_name=model_name)
 
 
-def run_all_in_one_upgrade(group, new_source, model_name=None):
+def run_all_in_one_upgrades(apps, new_source, model_name=None):
     """Upgrade payload of all applications in group using all-in-one method.
 
-    :param group: List of applications to upgrade.
-    :type group: []
+    :param apps: List of applications to upgrade.
+    :type apps: List[str]
     :source: New package origin.
     :type new_source: str
     :param model_name: Name of model to query.
     :type model_name: str
     """
     set_upgrade_application_config(
-        group,
+        apps,
         new_source,
         model_name=model_name,
         action_managed=False)
     zaza.model.block_until_all_units_idle()
 
 
-def run_upgrade(group, new_source, model_name=None):
+def run_upgrade_on_apps(apps, new_source, model_name=None):
     """Upgrade payload of all applications in group.
 
     Upgrade apps using action managed upgrades where possible and fallback to
     all_in_one method.
 
-    :param group: List of applications to upgrade.
-    :type group: []
+    :param apps: List of applications to upgrade.
+    :type apps: []
     :param new_source: New package origin.
     :type new_source: str
     :param model_name: Name of model to query.
     :type model_name: str
     """
-    action_upgrade = []
-    all_in_one_upgrade = []
-    for app in group:
+    action_upgrades = []
+    all_in_one_upgrades = []
+    for app in apps:
+        if is_already_upgraded(app, new_source, model_name=model_name):
+            logging.info("Application '%s' is already upgraded. Skipping.",
+                         app)
+            continue
         if is_action_upgradable(app, model_name=model_name):
-            action_upgrade.append(app)
+            action_upgrades.append(app)
         else:
-            all_in_one_upgrade.append(app)
-    run_all_in_one_upgrade(
-        all_in_one_upgrade,
-        new_source,
-        model_name=model_name)
-    run_action_upgrade(
-        action_upgrade,
-        new_source,
-        model_name=model_name)
+            all_in_one_upgrades.append(app)
+    if all_in_one_upgrades:
+        run_all_in_one_upgrades(
+            all_in_one_upgrades,
+            new_source,
+            model_name=model_name)
+    if action_upgrades:
+        run_action_upgrades(
+            action_upgrades,
+            new_source,
+            model_name=model_name)
 
 
 def run_upgrade_tests(new_source, model_name=None):
@@ -270,8 +338,6 @@ def run_upgrade_tests(new_source, model_name=None):
     :type model_name: str
     """
     groups = get_upgrade_groups(model_name=model_name)
-    run_upgrade(groups['Core Identity'], new_source, model_name=model_name)
-    run_upgrade(groups['Storage'], new_source, model_name=model_name)
-    run_upgrade(groups['Control Plane'], new_source, model_name=model_name)
-    run_upgrade(groups['Compute'], new_source, model_name=model_name)
-    run_upgrade(groups['sweep_up'], new_source, model_name=model_name)
+    for name, apps in groups:
+        logging.info("Performing upgrade of %s", name)
+        run_upgrade_on_apps(apps, new_source, model_name=model_name)

zaza/openstack/utilities/os_versions.py

@@ -36,6 +36,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('eoan', 'train'),
     ('focal', 'ussuri'),
     ('groovy', 'victoria'),
+    ('hirsute', 'wallaby'),
 ])
 
 
@@ -69,9 +70,9 @@ OPENSTACK_RELEASES_PAIRS = [
     'bionic_queens', 'bionic_rocky', 'cosmic_rocky',
     'bionic_stein', 'disco_stein', 'bionic_train',
     'eoan_train', 'bionic_ussuri', 'focal_ussuri',
-    'focal_victoria', 'groovy_victoria']
+    'focal_victoria', 'groovy_victoria',
+    'focal_wallaby', 'hirsute_wallaby']
 
-# The ugly duckling - must list releases oldest to newest
 SWIFT_CODENAMES = OrderedDict([
     ('diablo',
         ['1.4.3']),
@@ -113,6 +114,17 @@ SWIFT_CODENAMES = OrderedDict([
         ['2.25.0']),
 ])
 
+OVN_CODENAMES = OrderedDict([
+    ('train',
+        ['2.12']),
+    ('ussuri',
+        ['20.03']),
+    ('victoria',
+        ['20.06']),
+    ('wallaby',
+        ['20.12']),
+])
+
 # >= Liberty version->codename mapping
 PACKAGE_CODENAMES = {
     'nova-common': OrderedDict([
@@ -245,10 +257,6 @@ PACKAGE_CODENAMES = {
         ('10', 'ussuri'),
         ('11', 'victoria'),
     ]),
-    'ovn-common': OrderedDict([
-        ('2', 'train'),
-        ('20', 'ussuri'),
-    ]),
     'ceph-common': OrderedDict([
         ('10', 'mitaka'),  # jewel
         ('12', 'queens'),  # luminous
@@ -256,4 +264,9 @@ PACKAGE_CODENAMES = {
         ('14', 'train'),   # nautilus
         ('15', 'ussuri'),  # octopus
     ]),
+    'placement-common': OrderedDict([
+        ('2', 'train'),
+        ('3', 'ussuri'),
+        ('4', 'victoria'),
+    ]),
 }

zaza/openstack/utilities/parallel_series_upgrade.py

@@ -58,7 +58,9 @@ def app_config(charm_name):
     }
     exceptions = {
         'rabbitmq-server': {
-            'origin': 'source',
+            # NOTE: AJK disable config-changed on rabbitmq-server due to bug:
+            # #1896520
+            'origin': None,
             'pause_non_leader_subordinate': False,
             'post_application_upgrade_functions': [
                 ('zaza.openstack.charm_tests.rabbitmq_server.utils.'
@@ -94,7 +96,7 @@ def app_config(charm_name):
             'pause_non_leader_subordinate': True,
             'post_upgrade_functions': [
                 ('zaza.openstack.charm_tests.vault.setup.'
-                 'async_mojo_unseal_by_unit')]
+                 'async_mojo_or_default_unseal_by_unit')]
         },
         'mongodb': {
             'origin': None,
@@ -191,49 +193,64 @@ async def parallel_series_upgrade(
     status = (await model.async_get_status()).applications[application]
     logging.info(
         "Configuring leader / non leaders for {}".format(application))
-    leader, non_leaders = get_leader_and_non_leaders(status)
-    for leader_name, leader_unit in leader.items():
+    leaders, non_leaders = get_leader_and_non_leaders(status)
+    for leader_unit in leaders.values():
         leader_machine = leader_unit["machine"]
-        leader = leader_name
-    machines = [
-        unit["machine"] for name, unit
-        in non_leaders.items()
-        if unit['machine'] not in completed_machines]
+    machines = [unit["machine"] for name, unit in non_leaders.items()
+                if unit['machine'] not in completed_machines]
 
     await maybe_pause_things(
         status,
         non_leaders,
         pause_non_leader_subordinate,
         pause_non_leader_primary)
-    await series_upgrade_utils.async_set_series(
-        application, to_series=to_series)
-    app_idle = [
+    # wait for the entire application set to be idle before starting upgrades
+    await asyncio.gather(*[
         model.async_wait_for_unit_idle(unit, include_subordinates=True)
-        for unit in status["units"]
-    ]
-    await asyncio.gather(*app_idle)
+        for unit in status["units"]])
     await prepare_series_upgrade(leader_machine, to_series=to_series)
-    prepare_group = [
-        prepare_series_upgrade(machine, to_series=to_series)
-        for machine in machines]
-    await asyncio.gather(*prepare_group)
+    await asyncio.gather(*[
+        wait_for_idle_then_prepare_series_upgrade(
+            machine, to_series=to_series)
+        for machine in machines])
     if leader_machine not in completed_machines:
         machines.append(leader_machine)
-    upgrade_group = [
+    await asyncio.gather(*[
         series_upgrade_machine(
             machine,
             origin=origin,
             application=application,
             files=files, workaround_script=workaround_script,
             post_upgrade_functions=post_upgrade_functions)
-        for machine in machines
-    ]
-    await asyncio.gather(*upgrade_group)
+        for machine in machines])
     completed_machines.extend(machines)
+    await series_upgrade_utils.async_set_series(
+        application, to_series=to_series)
     await run_post_application_upgrade_functions(
         post_application_upgrade_functions)
 
 
+async def wait_for_idle_then_prepare_series_upgrade(
+        machine, to_series, model_name=None):
+    """Wait for the units to idle the do prepare_series_upgrade.
+
+    We need to be sure that all the units are idle prior to actually calling
+    prepare_series_upgrade() as otherwise the call will fail.  It has to be
+    checked because when the leader is paused it may kick off relation hooks in
+    the other units in an HA group.
+
+    :param machine: the machine that is going to be prepared
+    :type machine: str
+    :param to_series: The series to which to upgrade
+    :type to_series: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    """
+    await model.async_block_until_units_on_machine_are_idle(
+        machine, model_name=model_name)
+    await prepare_series_upgrade(machine, to_series=to_series)
+
+
 async def serial_series_upgrade(
     application,
     from_series='xenial',
@@ -307,8 +324,10 @@ async def serial_series_upgrade(
         non_leaders,
         pause_non_leader_subordinate,
         pause_non_leader_primary)
+    logging.info("Finishing pausing application: {}".format(application))
     await series_upgrade_utils.async_set_series(
         application, to_series=to_series)
+    logging.info("Finished set series for application: {}".format(application))
     if not follower_first and leader_machine not in completed_machines:
         await model.async_wait_for_unit_idle(leader, include_subordinates=True)
         await prepare_series_upgrade(leader_machine, to_series=to_series)
@@ -321,6 +340,8 @@ async def serial_series_upgrade(
             files=files, workaround_script=workaround_script,
             post_upgrade_functions=post_upgrade_functions)
         completed_machines.append(leader_machine)
+        logging.info("Finished upgrading of leader for application: {}"
+                     .format(application))
 
     # for machine in machines:
     for unit_name, unit in non_leaders.items():
@@ -339,6 +360,8 @@ async def serial_series_upgrade(
             files=files, workaround_script=workaround_script,
             post_upgrade_functions=post_upgrade_functions)
         completed_machines.append(machine)
+    logging.info("Finished upgrading non leaders for application: {}"
+                 .format(application))
 
     if follower_first and leader_machine not in completed_machines:
         await model.async_wait_for_unit_idle(leader, include_subordinates=True)
@@ -354,6 +377,7 @@ async def serial_series_upgrade(
         completed_machines.append(leader_machine)
     await run_post_application_upgrade_functions(
         post_application_upgrade_functions)
+    logging.info("Done series upgrade for: {}".format(application))
 
 
 async def series_upgrade_machine(
@@ -381,17 +405,16 @@ async def series_upgrade_machine(
     :returns: None
     :rtype: None
     """
-    logging.info(
-        "About to series-upgrade ({})".format(machine))
+    logging.info("About to series-upgrade ({})".format(machine))
     await run_pre_upgrade_functions(machine, pre_upgrade_functions)
     await add_confdef_file(machine)
     await async_dist_upgrade(machine)
     await async_do_release_upgrade(machine)
     await remove_confdef_file(machine)
     await reboot(machine)
+    await series_upgrade_utils.async_complete_series_upgrade(machine)
     if origin:
         await os_utils.async_set_origin(application, origin)
-    await series_upgrade_utils.async_complete_series_upgrade(machine)
     await run_post_upgrade_functions(post_upgrade_functions)
 
 
@@ -484,8 +507,7 @@ async def maybe_pause_things(
     :returns: Nothing
     :trype: None
     """
-    subordinate_pauses = []
-    leader_pauses = []
+    unit_pauses = []
     for unit in units:
         if pause_non_leader_subordinate:
             if status["units"][unit].get("subordinates"):
@@ -495,15 +517,19 @@ async def maybe_pause_things(
                         logging.info("Skipping pausing {} - blacklisted"
                                      .format(subordinate))
                     else:
-                        logging.info("Pausing {}".format(subordinate))
-                        subordinate_pauses.append(model.async_run_action(
-                            subordinate, "pause", action_params={}))
+                        unit_pauses.append(
+                            _pause_helper("subordinate", subordinate))
         if pause_non_leader_primary:
-            logging.info("Pausing {}".format(unit))
-            leader_pauses.append(
-                model.async_run_action(unit, "pause", action_params={}))
-    await asyncio.gather(*leader_pauses)
-    await asyncio.gather(*subordinate_pauses)
+            unit_pauses.append(_pause_helper("leader", unit))
+    if unit_pauses:
+        await asyncio.gather(*unit_pauses)
+
+
+async def _pause_helper(_type, unit):
+    """Pause helper to ensure that the log happens nearer to the action."""
+    logging.info("Pausing ({}) {}".format(_type, unit))
+    await model.async_run_action(unit, "pause", action_params={})
+    logging.info("Finished Pausing ({}) {}".format(_type, unit))
 
 
 def get_leader_and_non_leaders(status):
@@ -541,14 +567,14 @@ async def prepare_series_upgrade(machine, to_series):
     NOTE: This is a new feature in juju behind a feature flag and not yet in
     libjuju.
     export JUJU_DEV_FEATURE_FLAGS=upgrade-series
-    :param machine_num: Machine number
-    :type machine_num: str
+    :param machine: Machine number
+    :type machine: str
     :param to_series: The series to which to upgrade
     :type to_series: str
     :returns: None
     :rtype: None
     """
-    logging.info("Preparing series upgrade for: {}".format(machine))
+    logging.info("Preparing series upgrade for: %s", machine)
     await series_upgrade_utils.async_prepare_series_upgrade(
         machine, to_series=to_series)
 
@@ -564,9 +590,8 @@ async def reboot(machine):
     try:
         await model.async_run_on_machine(machine, 'sudo init 6 & exit')
         # await run_on_machine(unit, "sudo reboot && exit")
-    except subprocess.CalledProcessError as e:
-        logging.warn("Error doing reboot: {}".format(e))
-        pass
+    except subprocess.CalledProcessError as error:
+        logging.warning("Error doing reboot: %s", error)
 
 
 async def async_dist_upgrade(machine):
@@ -577,16 +602,31 @@ async def async_dist_upgrade(machine):
     :returns: None
     :rtype: None
     """
-    logging.info('Updating package db ' + machine)
+    logging.info('Updating package db %s', machine)
     update_cmd = 'sudo apt-get update'
     await model.async_run_on_machine(machine, update_cmd)
 
-    logging.info('Updating existing packages ' + machine)
+    logging.info('Updating existing packages %s', machine)
     dist_upgrade_cmd = (
         """yes | sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes """
         """-o "Dpkg::Options::=--force-confdef" """
         """-o "Dpkg::Options::=--force-confold" dist-upgrade""")
     await model.async_run_on_machine(machine, dist_upgrade_cmd)
+    rdict = await model.async_run_on_machine(
+        machine,
+        "cat /var/run/reboot-required || true")
+    if "Stdout" in rdict and "restart" in rdict["Stdout"].lower():
+        logging.info("dist-upgrade required reboot machine: %s", machine)
+        await reboot(machine)
+        logging.info("Waiting for machine to come back afer reboot: %s",
+                     machine)
+        await model.async_block_until_file_missing_on_machine(
+            machine, "/var/run/reboot-required")
+        logging.info("Waiting for machine idleness on %s", machine)
+        await asyncio.sleep(5.0)
+        await model.async_block_until_units_on_machine_are_idle(machine)
+        # TODO: change this to wait on units on the machine
+        # await model.async_block_until_all_units_idle()
 
 
 async def async_do_release_upgrade(machine):
@@ -597,7 +637,7 @@ async def async_do_release_upgrade(machine):
     :returns: None
     :rtype: None
     """
-    logging.info('Upgrading ' + machine)
+    logging.info('Upgrading %s', machine)
     do_release_upgrade_cmd = (
         'yes | sudo DEBIAN_FRONTEND=noninteractive '
         'do-release-upgrade -f DistUpgradeViewNonInteractive')

zaza/openstack/utilities/series_upgrade.py

@@ -884,7 +884,8 @@ def dist_upgrade(unit_name):
         """-o "Dpkg::Options::=--force-confdef" """
         """-o "Dpkg::Options::=--force-confold" dist-upgrade""")
     model.run_on_unit(unit_name, dist_upgrade_cmd)
-    rdict = model.run_on_unit(unit_name, "cat /var/run/reboot-required")
+    rdict = model.run_on_unit(unit_name,
+                              "cat /var/run/reboot-required || true")
     if "Stdout" in rdict and "restart" in rdict["Stdout"].lower():
         logging.info("dist-upgrade required reboot {}".format(unit_name))
         os_utils.reboot(unit_name)
@@ -919,8 +920,8 @@ async def async_dist_upgrade(unit_name):
         """-o "Dpkg::Options::=--force-confdef" """
         """-o "Dpkg::Options::=--force-confold" dist-upgrade""")
     await model.async_run_on_unit(unit_name, dist_upgrade_cmd)
-    rdict = await model.async_run_on_unit(unit_name,
-                                          "cat /var/run/reboot-required")
+    rdict = await model.async_run_on_unit(
+        unit_name, "cat /var/run/reboot-required || true")
     if "Stdout" in rdict and "restart" in rdict["Stdout"].lower():
         logging.info("dist-upgrade required reboot {}".format(unit_name))
         await os_utils.async_reboot(unit_name)

zaza/openstack/utilities/upgrade_utils.py

@@ -19,8 +19,20 @@ import logging
 import re
 
 import zaza.model
+import zaza.utilities.juju
+from zaza.openstack.utilities.os_versions import (
+    OPENSTACK_CODENAMES,
+    UBUNTU_OPENSTACK_RELEASE,
+    OPENSTACK_RELEASES_PAIRS,
+)
 
-
+"""
+The below upgrade order is surfaced in end-user documentation. Any change to
+it should be accompanied by an update to the OpenStack Charms Deployment Guide
+for both charm upgrades and payload upgrades:
+- source/upgrade-charms.rst#upgrade-order
+- source/upgrade-openstack.rst#openstack_upgrade_order
+"""
 SERVICE_GROUPS = (
     ('Database Services', ['percona-cluster', 'mysql-innodb-cluster']),
     ('Stateful Services', ['rabbitmq-server', 'ceph-mon']),
@@ -46,7 +58,7 @@ def get_upgrade_candidates(model_name=None, filters=None):
     :param filters: List of filter functions to apply
     :type filters: List[fn]
     :returns: List of application that can have their payload upgraded.
-    :rtype: []
+    :rtype: Dict[str, Dict[str, ANY]]
     """
     if filters is None:
         filters = []
@@ -163,8 +175,8 @@ def get_series_upgrade_groups(model_name=None, extra_filters=None):
 
     :param model_name: Name of model to query.
     :type model_name: str
-    :returns: Dict of group lists keyed on group name.
-    :rtype: collections.OrderedDict
+    :returns: List of tuples(group name, applications)
+    :rtype: List[Tuple[str, Dict[str, ANY]]]
     """
     filters = [_filter_subordinates]
     filters = _apply_extra_filters(filters, extra_filters)
@@ -226,3 +238,110 @@ def extract_charm_name_from_url(charm_url):
     """
     charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1])
     return charm_name.split(':')[-1]
+
+
+def get_all_principal_applications(model_name=None):
+    """Return a list of all the prinical applications in the model.
+
+    :param model_name: Optional model name
+    :type model_name: Optional[str]
+    :returns: List of principal application names
+    :rtype: List[str]
+    """
+    status = zaza.utilities.juju.get_full_juju_status(model_name=model_name)
+    return [application for application in status.applications.keys()
+            if not status.applications.get(application)['subordinate-to']]
+
+
+def get_lowest_openstack_version(current_versions):
+    """Get the lowest OpenStack version from the list of current versions.
+
+    :param current_versions: The list of versions
+    :type current_versions: List[str]
+    :returns: the lowest version currently installed.
+    :rtype: str
+    """
+    lowest_version = 'zebra'
+    for svc in current_versions.keys():
+        if current_versions[svc] < lowest_version:
+            lowest_version = current_versions[svc]
+    return lowest_version
+
+
+def determine_next_openstack_release(release):
+    """Determine the next release after the one passed as a str.
+
+    The returned value is a tuple of the form: ('2020.1', 'ussuri')
+
+    :param release: the release to use as the base
+    :type release: str
+    :returns: the release tuple immediately after the current one.
+    :rtype: Tuple[str, str]
+    :raises: KeyError if the current release doesn't actually exist
+    """
+    old_index = list(OPENSTACK_CODENAMES.values()).index(release)
+    new_index = old_index + 1
+    return list(OPENSTACK_CODENAMES.items())[new_index]
+
+
+def determine_new_source(ubuntu_version, current_source, new_release,
+                         single_increment=True):
+    """Determine the new source/openstack-origin value  based on new release.
+
+    This takes the ubuntu_version and the current_source (in the form of
+    'distro' or 'cloud:xenial-mitaka') and either converts it to a new source,
+    or returns None if the new_release will match the current_source (i.e. it's
+    already at the right release), or it's simply not possible.
+
+    If single_increment is set, then the returned source will only be returned
+    if the new_release is one more than the release in the current source.
+
+    :param ubuntu_version: the ubuntu version that the app is installed on.
+    :type ubuntu_version: str
+    :param current_source: a source in the form of 'distro' or
+        'cloud:xenial-mitaka'
+    :type current_source: str
+    :param new_release: a new OpenStack version codename. e.g. 'stein'
+    :type new_release: str
+    :param single_increment: If True, only allow single increment upgrade.
+    :type single_increment: boolean
+    :returns: The new source in the form of 'cloud:bionic-train' or None if not
+        possible
+    :rtype: Optional[str]
+    :raises: KeyError if any of the strings don't correspond to known values.
+    """
+    logging.warn("determine_new_source: locals: %s", locals())
+    if current_source == 'distro':
+        # convert to a ubuntu-openstack pair
+        current_source = "cloud:{}-{}".format(
+            ubuntu_version, UBUNTU_OPENSTACK_RELEASE[ubuntu_version])
+    # strip out the current openstack version
+    if ':' not in current_source:
+        current_source = "cloud:{}-{}".format(ubuntu_version, current_source)
+    pair = current_source.split(':')[1]
+    u_version, os_version = pair.split('-', 2)
+    if u_version != ubuntu_version:
+        logging.warn("determine_new_source: ubuntu_versions don't match: "
+                     "%s != %s" % (ubuntu_version, u_version))
+        return None
+    # determine versions
+    openstack_codenames = list(OPENSTACK_CODENAMES.values())
+    old_index = openstack_codenames.index(os_version)
+    try:
+        new_os_version = openstack_codenames[old_index + 1]
+    except IndexError:
+        logging.warn("determine_new_source: no OpenStack version after "
+                     "'%s'" % os_version)
+        return None
+    if single_increment and new_release != new_os_version:
+        logging.warn("determine_new_source: requested version '%s' not a "
+                     "single increment from '%s' which is '%s'" % (
+                         new_release, os_version, new_os_version))
+        return None
+    # now check that there is a combination of u_version-new_os_version
+    new_pair = "{}_{}".format(u_version, new_os_version)
+    if new_pair not in OPENSTACK_RELEASES_PAIRS:
+        logging.warn("determine_new_source: now release pair candidate for "
+                     " combination cloud:%s-%s" % (u_version, new_os_version))
+        return None
+    return "cloud:{}-{}".format(u_version, new_os_version)