Update openstack upgrade tests for focal (#476)

This patch modifies the existing openstack upgrade tests so that they
work with focal (by explicitly supporting mysql-innodb-cluster), and are
also interruptable and resumable (at a charm level).  It also makes them
work with the udpated 'get_upgrade_groups()' that ultimately gets a List
of Tuples rather than a dictionary.

unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py

@@ -44,25 +44,37 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
         self.patch_object(
             openstack_upgrade.zaza.model,
             "get_application_config")
+        self.patch_object(
+            openstack_upgrade.zaza.model,
+            "block_until_all_units_idle")
+        self.patch_object(
+            openstack_upgrade,
+            "block_until_mysql_innodb_cluster_has_rw")
 
         def _get_application_config(app, model_name=None):
             app_config = {
-                'ceph-mon': {'verbose': True, 'source': 'old-src'},
-                'neutron-openvswitch': {'verbose': True},
-                'ntp': {'verbose': True},
-                'percona-cluster': {'verbose': True, 'source': 'old-src'},
+                'ceph-mon': {'verbose': {'value': True},
+                             'source': {'value': 'old-src'}},
+                'neutron-openvswitch': {'verbose': {'value': True}},
+                'ntp': {'verbose': {'value': True}},
+                'percona-cluster': {'verbose': {'value': True},
+                                    'source': {'value': 'old-src'}},
                 'cinder': {
-                    'verbose': True,
-                    'openstack-origin': 'old-src',
-                    'action-managed-upgrade': False},
+                    'verbose': {'value': True},
+                    'openstack-origin': {'value': 'old-src'},
+                    'action-managed-upgrade': {'value': False}},
                 'neutron-api': {
-                    'verbose': True,
-                    'openstack-origin': 'old-src',
-                    'action-managed-upgrade': False},
+                    'verbose': {'value': True},
+                    'openstack-origin': {'value': 'old-src'},
+                    'action-managed-upgrade': {'value': False}},
                 'nova-compute': {
-                    'verbose': True,
-                    'openstack-origin': 'old-src',
-                    'action-managed-upgrade': False},
+                    'verbose': {'value': True},
+                    'openstack-origin': {'value': 'old-src'},
+                    'action-managed-upgrade': {'value': False}},
+                'mysql-innodb-cluster': {
+                    'verbose': {'value': True},
+                    'source': {'value': 'old-src'},
+                    'action-managed-upgrade': {'value': True}},
             }
             return app_config[app]
         self.get_application_config.side_effect = _get_application_config
@@ -74,6 +86,10 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
                 'subordinate-to': 'nova-compute'},
             'ntp': {  # Filter as it has no source option
                 'charm': 'cs:ntp'},
+            'mysql-innodb-cluster': {
+                'charm': 'cs:mysql-innodb-cluster',
+                'units': {
+                    'mysql-innodb-cluster/0': {}}},
             'nova-compute': {
                 'charm': 'cs:nova-compute',
                 'units': {
@@ -115,7 +131,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
             model_name=None,
             raise_on_failure=True)
 
-    def test_action_upgrade_group(self):
+    def test_action_upgrade_apps(self):
         self.patch_object(openstack_upgrade, "pause_units")
         self.patch_object(openstack_upgrade, "action_unit_upgrade")
         self.patch_object(openstack_upgrade, "resume_units")
@@ -127,7 +143,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
             'nova-compute': [mock_nova_compute_0],
             'cinder': [mock_cinder_1]}
         self.get_units.side_effect = lambda app, model_name: units[app]
-        openstack_upgrade.action_upgrade_group(['nova-compute', 'cinder'])
+        openstack_upgrade.action_upgrade_apps(['nova-compute', 'cinder'])
         pause_calls = [
             mock.call(['cinder-hacluster/0'], model_name=None),
             mock.call(['nova-compute/0', 'cinder/1'], model_name=None)]
@@ -142,6 +158,30 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
             mock.call(['cinder-hacluster/0'], model_name=None)]
         self.resume_units.assert_has_calls(resume_calls, any_order=False)
 
+    def test_action_upgrade_apps_mysql_innodb_cluster(self):
+        """Verify that mysql-innodb-cluster is settled before complete."""
+        self.patch_object(openstack_upgrade, "pause_units")
+        self.patch_object(openstack_upgrade, "action_unit_upgrade")
+        self.patch_object(openstack_upgrade, "resume_units")
+        mock_mysql_innodb_cluster_0 = mock.MagicMock()
+        mock_mysql_innodb_cluster_0.entity_id = 'mysql-innodb-cluster/0'
+        units = {'mysql-innodb-cluster': [mock_mysql_innodb_cluster_0]}
+        self.get_units.side_effect = lambda app, model_name: units[app]
+        openstack_upgrade.action_upgrade_apps(['mysql-innodb-cluster'])
+        pause_calls = [
+            mock.call(['mysql-innodb-cluster/0'], model_name=None)]
+        self.pause_units.assert_has_calls(pause_calls, any_order=False)
+        action_unit_upgrade_calls = [
+            mock.call(['mysql-innodb-cluster/0'], model_name=None)]
+        self.action_unit_upgrade.assert_has_calls(
+            action_unit_upgrade_calls,
+            any_order=False)
+        resume_calls = [
+            mock.call(['mysql-innodb-cluster/0'], model_name=None)]
+        self.resume_units.assert_has_calls(resume_calls, any_order=False)
+        self.block_until_mysql_innodb_cluster_has_rw.assert_called_once_with(
+            None)
+
     def test_set_upgrade_application_config(self):
         openstack_upgrade.set_upgrade_application_config(
             ['neutron-api', 'cinder'],
@@ -177,17 +217,23 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
         self.assertFalse(
             openstack_upgrade.is_action_upgradable('percona-cluster'))
 
+    def test_is_already_upgraded(self):
+        self.assertTrue(
+            openstack_upgrade.is_already_upgraded('cinder', 'old-src'))
+        self.assertFalse(
+            openstack_upgrade.is_already_upgraded('cinder', 'new-src'))
+
     def test_run_action_upgrade(self):
         self.patch_object(openstack_upgrade, "set_upgrade_application_config")
-        self.patch_object(openstack_upgrade, "action_upgrade_group")
-        openstack_upgrade.run_action_upgrade(
+        self.patch_object(openstack_upgrade, "action_upgrade_apps")
+        openstack_upgrade.run_action_upgrades(
             ['cinder', 'neutron-api'],
             'new-src')
         self.set_upgrade_application_config.assert_called_once_with(
             ['cinder', 'neutron-api'],
             'new-src',
             model_name=None)
-        self.action_upgrade_group.assert_called_once_with(
+        self.action_upgrade_apps.assert_called_once_with(
             ['cinder', 'neutron-api'],
             model_name=None)
 
@@ -196,7 +242,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
         self.patch_object(
             openstack_upgrade.zaza.model,
             'block_until_all_units_idle')
-        openstack_upgrade.run_all_in_one_upgrade(
+        openstack_upgrade.run_all_in_one_upgrades(
             ['percona-cluster'],
             'new-src')
         self.set_upgrade_application_config.assert_called_once_with(
@@ -207,34 +253,36 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
         self.block_until_all_units_idle.assert_called_once_with()
 
     def test_run_upgrade(self):
-        self.patch_object(openstack_upgrade, "run_all_in_one_upgrade")
-        self.patch_object(openstack_upgrade, "run_action_upgrade")
-        openstack_upgrade.run_upgrade(
+        self.patch_object(openstack_upgrade, "run_all_in_one_upgrades")
+        self.patch_object(openstack_upgrade, "run_action_upgrades")
+        openstack_upgrade.run_upgrade_on_apps(
             ['cinder', 'neutron-api', 'ceph-mon'],
             'new-src')
-        self.run_all_in_one_upgrade.assert_called_once_with(
+        self.run_all_in_one_upgrades.assert_called_once_with(
             ['ceph-mon'],
             'new-src',
             model_name=None)
-        self.run_action_upgrade.assert_called_once_with(
+        self.run_action_upgrades.assert_called_once_with(
             ['cinder', 'neutron-api'],
             'new-src',
             model_name=None)
 
     def test_run_upgrade_tests(self):
-        self.patch_object(openstack_upgrade, "run_upgrade")
+        self.patch_object(openstack_upgrade, "run_upgrade_on_apps")
         self.patch_object(openstack_upgrade, "get_upgrade_groups")
-        self.get_upgrade_groups.return_value = {
-            'Compute': ['nova-compute'],
-            'Control Plane': ['cinder', 'neutron-api'],
-            'Core Identity': ['keystone'],
-            'Storage': ['ceph-mon'],
-            'sweep_up': ['designate']}
+        self.get_upgrade_groups.return_value = [
+            ('Compute', ['nova-compute']),
+            ('Control Plane', ['cinder', 'neutron-api']),
+            ('Core Identity', ['keystone']),
+            ('Storage', ['ceph-mon']),
+            ('sweep_up', ['designate'])]
         openstack_upgrade.run_upgrade_tests('new-src', model_name=None)
         run_upgrade_calls = [
+            mock.call(['nova-compute'], 'new-src', model_name=None),
+            mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None),
             mock.call(['keystone'], 'new-src', model_name=None),
             mock.call(['ceph-mon'], 'new-src', model_name=None),
-            mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None),
-            mock.call(['nova-compute'], 'new-src', model_name=None),
-            mock.call(['designate'], 'new-src', model_name=None)]
-        self.run_upgrade.assert_has_calls(run_upgrade_calls, any_order=False)
+            mock.call(['designate'], 'new-src', model_name=None),
+        ]
+        self.run_upgrade_on_apps.assert_has_calls(
+            run_upgrade_calls, any_order=False)

zaza/openstack/charm_tests/openstack_upgrade/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Code for testing openstack upgrades."""

zaza/openstack/charm_tests/openstack_upgrade/tests.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Define class for OpenStack Upgrade."""
+
+import logging
+import unittest
+
+from zaza.openstack.utilities import (
+    cli as cli_utils,
+    upgrade_utils as upgrade_utils,
+    openstack as openstack_utils,
+    openstack_upgrade as openstack_upgrade,
+)
+from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest
+
+
+class OpenStackUpgradeVMLaunchBase(object):
+    """A base class to peform a simple validation on the cloud.
+
+    This wraps an OpenStack upgrade with a VM launch before and after the
+    upgrade.
+
+    This test requires a full OpenStack including at least: keystone, glance,
+    nova-cloud-controller, nova-compute, neutron-gateway, neutron-api and
+    neutron-openvswitch.
+
+    This class should be used as a base class to the upgrade 'test'.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for OpenStack Upgrades."""
+        print("Running OpenStackUpgradeMixin setUpClass")
+        super().setUpClass()
+        cls.lts = LTSGuestCreateTest()
+        cls.lts.setUpClass()
+
+    def test_100_validate_pre_openstack_upgrade_cloud(self):
+        """Validate pre openstack upgrade."""
+        logging.info("Validate pre-openstack-upgrade: Spin up LTS instance")
+        self.lts.test_launch_small_instance()
+
+    def test_500_validate_openstack_upgraded_cloud(self):
+        """Validate post openstack upgrade."""
+        logging.info("Validate post-openstack-upgrade: Spin up LTS instance")
+        self.lts.test_launch_small_instance()
+
+
+class WaitForMySQL(unittest.TestCase):
+    """Helper test to wait on mysql-innodb-cluster to be fully ready.
+
+    In practice this means that there is at least on R/W unit available.
+    Sometimes, after restarting units in the mysql-innodb-cluster, all the
+    units are R/O until the cluster picks the R/W unit.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """Set up class."""
+        print("Running OpenstackUpgradeTests setUpClass")
+        super().setUpClass()
+        cli_utils.setup_logging()
+
+    def test_100_wait_for_happy_mysql_innodb_cluster(self):
+        """Wait for mysql cluster to have at least one R/W node."""
+        logging.info("Starting wait for an R/W unit.")
+        openstack_upgrade.block_until_mysql_innodb_cluster_has_rw()
+        logging.info("Done .. all seems well.")
+
+
+class OpenStackUpgradeTestsFocalUssuri(OpenStackUpgradeVMLaunchBase):
+    """Upgrade OpenStack from distro -> cloud:focal-victoria."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for OpenStack Upgrades."""
+        print("Running OpenstackUpgradeTests setUpClass")
+        super().setUpClass()
+        cli_utils.setup_logging()
+
+    def test_200_run_openstack_upgrade(self):
+        """Run openstack upgrade, but work out what to do."""
+        openstack_upgrade.run_upgrade_tests("cloud:focal-victoria")
+
+
+class OpenStackUpgradeTests(OpenStackUpgradeVMLaunchBase):
+    """A Principal Class to encapsulate OpenStack Upgrade Tests.
+
+    A generic Test class that can discover which Ubuntu version and OpenStack
+    version to upgrade from.
+
+    TODO: Not used at present.  Use the declarative tests directly that choose
+    the version to upgrade to.  The functions that this class depends on need a
+    bit more work regarding how the determine which version to go to.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for OpenStack Upgrades."""
+        print("Running OpenstackUpgradeTests setUpClass")
+        super().setUpClass()
+        cli_utils.setup_logging()
+
+    def test_200_run_openstack_upgrade(self):
+        """Run openstack upgrade, but work out what to do.
+
+        TODO: This is really inefficient at the moment, and doesn't (yet)
+        determine which ubuntu version to work from.  Don't use until we can
+        make it better.
+        """
+        # TODO: work out the most recent Ubuntu version; we assume this is the
+        # version that OpenStack is running on.
+        ubuntu_version = "focal"
+        logging.info("Getting all principle applications ...")
+        principle_services = upgrade_utils.get_all_principal_applications()
+        logging.info(
+            "Getting OpenStack vesions from principal applications ...")
+        current_versions = openstack_utils.get_current_os_versions(
+            principle_services)
+        logging.info("current versions: %s" % current_versions)
+        # Find the lowest value openstack release across all services and make
+        # sure all servcies are upgraded to one release higher than the lowest
+        from_version = upgrade_utils.get_lowest_openstack_version(
+            current_versions)
+        logging.info("from version: %s" % from_version)
+        to_version = upgrade_utils.determine_next_openstack_release(
+            from_version)[1]
+        logging.info("to version: %s" % to_version)
+        # TODO: need to determine the ubuntu base verion that is being upgraded
+        target_source = upgrade_utils.determine_new_source(
+            ubuntu_version, from_version, to_version, single_increment=True)
+        logging.info("target source: %s" % target_source)
+        assert target_source is not None
+        openstack_upgrade.run_upgrade_tests(target_source)

zaza/openstack/charm_tests/vault/setup.py

@@ -174,7 +174,8 @@ def auto_initialize(cacert=None, validation_application='keystone', wait=True):
         zaza.model.wait_for_agent_status()
         test_config = lifecycle_utils.get_charm_config(fatal=False)
         zaza.model.wait_for_application_states(
-            states=test_config.get('target_deploy_status', {}))
+            states=test_config.get('target_deploy_status', {}),
+            timeout=7200)
 
     if validation_application:
         validate_ca(cacertificate, application=validation_application)

zaza/openstack/utilities/openstack.py

@@ -16,6 +16,23 @@
 
 This module contains a number of functions for interacting with OpenStack.
 """
+import datetime
+import io
+import itertools
+import juju_wait
+import logging
+import os
+import paramiko
+import re
+import six
+import subprocess
+import sys
+import tempfile
+import tenacity
+import textwrap
+import urllib
+
+
 from .os_versions import (
     OPENSTACK_CODENAMES,
     SWIFT_CODENAMES,
@@ -48,21 +65,6 @@ from neutronclient.common import exceptions as neutronexceptions
 from octaviaclient.api.v2 import octavia as octaviaclient
 from swiftclient import client as swiftclient
 
-import datetime
-import io
-import itertools
-import juju_wait
-import logging
-import os
-import paramiko
-import re
-import six
-import subprocess
-import sys
-import tempfile
-import tenacity
-import textwrap
-import urllib
 
 import zaza
 
@@ -1554,6 +1556,7 @@ def get_current_os_versions(deployed_applications, model_name=None):
     for application in UPGRADE_SERVICES:
         if application['name'] not in deployed_applications:
             continue
+        logging.info("looking at application: {}".format(application))
 
         version = generic_utils.get_pkg_version(application['name'],
                                                 application['type']['pkg'],

zaza/openstack/utilities/openstack_upgrade.py

@@ -95,8 +95,8 @@ async def async_action_unit_upgrade(units, model_name=None):
 action_unit_upgrade = sync_wrapper(async_action_unit_upgrade)
 
 
-def action_upgrade_group(applications, model_name=None):
-    """Upgrade units using action managed upgrades.
+def action_upgrade_apps(applications, model_name=None):
+    """Upgrade units in the applications using action managed upgrades.
 
     Upgrade all units of the given applications using action managed upgrades.
     This involves the following process:
@@ -142,6 +142,45 @@ def action_upgrade_group(applications, model_name=None):
 
         done.extend(target)
 
+    # Ensure that mysql-innodb-cluster has at least one R/W group (it can get
+    # into a state where all are R/O whilst it is sorting itself out after an
+    # openstack_upgrade
+    if "mysql-innodb-cluster" in applications:
+        block_until_mysql_innodb_cluster_has_rw(model_name)
+
+    # Now we need to wait for the model to go back to idle.
+    zaza.model.block_until_all_units_idle(model_name)
+
+
+async def async_block_until_mysql_innodb_cluster_has_rw(model=None,
+                                                        timeout=None):
+    """Block until the mysql-innodb-cluster is in a healthy state.
+
+    Curiously, after a series of pauses and restarts (e.g. during an upgrade)
+    the mysql-innodb-cluster charms may not yet have agreed which one is the
+    R/W node; i.e. they are all R/O.  Anyway, eventually they sort it out and
+    one jumps to the front and says "it's me!".  This is detected, externally,
+    by the status line including R/W in the output.
+
+    This function blocks until that happens so that no charm attempts to have a
+    chat with the mysql server before it has settled, thus breaking the whole
+    test.
+    """
+    async def async_check_workload_messages_for_rw(model=None):
+        """Return True if a least one work message contains R/W."""
+        status = await zaza.model.async_get_status()
+        app_status = status.applications.get("mysql-innodb-cluster")
+        units_data = app_status.units.values()
+        workload_statuses = [d.workload_status.info for d in units_data]
+        return any("R/W" in s for s in workload_statuses)
+
+    await zaza.model.async_block_until(async_check_workload_messages_for_rw,
+                                       timeout=timeout)
+
+
+block_until_mysql_innodb_cluster_has_rw = sync_wrapper(
+    async_block_until_mysql_innodb_cluster_has_rw)
+
 
 def set_upgrade_application_config(applications, new_source,
                                    action_managed=True, model_name=None):
@@ -150,7 +189,7 @@ def set_upgrade_application_config(applications, new_source,
     Set the charm config for upgrade.
 
     :param applications: List of application names.
-    :type applications: []
+    :type applications: List[str]
     :param new_source: New package origin.
     :type new_source: str
     :param action_managed: Whether to set action-managed-upgrade config option.
@@ -180,8 +219,8 @@ def set_upgrade_application_config(applications, new_source,
 def is_action_upgradable(app, model_name=None):
     """Can application be upgraded using action managed upgrade method.
 
-    :param new_source: New package origin.
-    :type new_source: str
+    :param app: The application to check
+    :type app: str
     :param model_name: Name of model to query.
     :type model_name: str
     :returns: Whether app be upgraded using action managed upgrade method.
@@ -196,66 +235,95 @@ def is_action_upgradable(app, model_name=None):
     return supported
 
 
-def run_action_upgrade(group, new_source, model_name=None):
+def is_already_upgraded(app, new_src, model_name=None):
+    """Return True if the app has already been upgraded.
+
+    :param app: The application to check
+    :type app: str
+    :param new_src: the new source (distro, cloud:x-y, etc.)
+    :type new_src: str
+    :param model_name: Name of model to query.
+    :type model_name: str
+    :returns: Whether app be upgraded using action managed upgrade method.
+    :rtype: bool
+    """
+    config = zaza.model.get_application_config(app, model_name=model_name)
+    try:
+        src = config['openstack-origin']['value']
+        key_was = 'openstack-origin'
+    except KeyError:
+        src = config['source']['value']
+        key_was = 'source'
+    logging.info("origin for {} is {}={}".format(app, key_was, src))
+    return src == new_src
+
+
+def run_action_upgrades(apps, new_source, model_name=None):
     """Upgrade payload of all applications in group using action upgrades.
 
-    :param group: List of applications to upgrade.
-    :type group
+    :param apps: List of applications to upgrade.
+    :type apps: List[str]
     :param new_source: New package origin.
     :type new_source: str
     :param model_name: Name of model to query.
     :type model_name: str
     """
-    set_upgrade_application_config(group, new_source, model_name=model_name)
-    action_upgrade_group(group, model_name=model_name)
+    set_upgrade_application_config(apps, new_source, model_name=model_name)
+    action_upgrade_apps(apps, model_name=model_name)
 
 
-def run_all_in_one_upgrade(group, new_source, model_name=None):
+def run_all_in_one_upgrades(apps, new_source, model_name=None):
     """Upgrade payload of all applications in group using all-in-one method.
 
-    :param group: List of applications to upgrade.
-    :type group: []
+    :param apps: List of applications to upgrade.
+    :type apps: List[str]
     :source: New package origin.
     :type new_source: str
     :param model_name: Name of model to query.
     :type model_name: str
     """
     set_upgrade_application_config(
-        group,
+        apps,
         new_source,
         model_name=model_name,
         action_managed=False)
     zaza.model.block_until_all_units_idle()
 
 
-def run_upgrade(group, new_source, model_name=None):
+def run_upgrade_on_apps(apps, new_source, model_name=None):
     """Upgrade payload of all applications in group.
 
     Upgrade apps using action managed upgrades where possible and fallback to
     all_in_one method.
 
-    :param group: List of applications to upgrade.
-    :type group: []
+    :param apps: List of applications to upgrade.
+    :type apps: []
     :param new_source: New package origin.
     :type new_source: str
     :param model_name: Name of model to query.
     :type model_name: str
     """
-    action_upgrade = []
-    all_in_one_upgrade = []
-    for app in group:
+    action_upgrades = []
+    all_in_one_upgrades = []
+    for app in apps:
+        if is_already_upgraded(app, new_source, model_name=model_name):
+            logging.info("Application '%s' is already upgraded. Skipping.",
+                         app)
+            continue
         if is_action_upgradable(app, model_name=model_name):
-            action_upgrade.append(app)
+            action_upgrades.append(app)
         else:
-            all_in_one_upgrade.append(app)
-    run_all_in_one_upgrade(
-        all_in_one_upgrade,
-        new_source,
-        model_name=model_name)
-    run_action_upgrade(
-        action_upgrade,
-        new_source,
-        model_name=model_name)
+            all_in_one_upgrades.append(app)
+    if all_in_one_upgrades:
+        run_all_in_one_upgrades(
+            all_in_one_upgrades,
+            new_source,
+            model_name=model_name)
+    if action_upgrades:
+        run_action_upgrades(
+            action_upgrades,
+            new_source,
+            model_name=model_name)
 
 
 def run_upgrade_tests(new_source, model_name=None):
@@ -270,8 +338,6 @@ def run_upgrade_tests(new_source, model_name=None):
     :type model_name: str
     """
     groups = get_upgrade_groups(model_name=model_name)
-    run_upgrade(groups['Core Identity'], new_source, model_name=model_name)
-    run_upgrade(groups['Storage'], new_source, model_name=model_name)
-    run_upgrade(groups['Control Plane'], new_source, model_name=model_name)
-    run_upgrade(groups['Compute'], new_source, model_name=model_name)
-    run_upgrade(groups['sweep_up'], new_source, model_name=model_name)
+    for name, apps in groups:
+        logging.info("Performing upgrade of %s", name)
+        run_upgrade_on_apps(apps, new_source, model_name=model_name)

zaza/openstack/utilities/upgrade_utils.py

@@ -19,6 +19,12 @@ import logging
 import re
 
 import zaza.model
+import zaza.utilities.juju
+from zaza.openstack.utilities.os_versions import (
+    OPENSTACK_CODENAMES,
+    UBUNTU_OPENSTACK_RELEASE,
+    OPENSTACK_RELEASES_PAIRS,
+)
 
 
 SERVICE_GROUPS = (
@@ -46,7 +52,7 @@ def get_upgrade_candidates(model_name=None, filters=None):
     :param filters: List of filter functions to apply
     :type filters: List[fn]
     :returns: List of application that can have their payload upgraded.
-    :rtype: []
+    :rtype: Dict[str, Dict[str, ANY]]
     """
     if filters is None:
         filters = []
@@ -163,8 +169,8 @@ def get_series_upgrade_groups(model_name=None, extra_filters=None):
 
     :param model_name: Name of model to query.
     :type model_name: str
-    :returns: Dict of group lists keyed on group name.
-    :rtype: collections.OrderedDict
+    :returns: List of tuples(group name, applications)
+    :rtype: List[Tuple[str, Dict[str, ANY]]]
     """
     filters = [_filter_subordinates]
     filters = _apply_extra_filters(filters, extra_filters)
@@ -226,3 +232,110 @@ def extract_charm_name_from_url(charm_url):
     """
     charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1])
     return charm_name.split(':')[-1]
+
+
+def get_all_principal_applications(model_name=None):
+    """Return a list of all the prinical applications in the model.
+
+    :param model_name: Optional model name
+    :type model_name: Optional[str]
+    :returns: List of principal application names
+    :rtype: List[str]
+    """
+    status = zaza.utilities.juju.get_full_juju_status(model_name=model_name)
+    return [application for application in status.applications.keys()
+            if not status.applications.get(application)['subordinate-to']]
+
+
+def get_lowest_openstack_version(current_versions):
+    """Get the lowest OpenStack version from the list of current versions.
+
+    :param current_versions: The list of versions
+    :type current_versions: List[str]
+    :returns: the lowest version currently installed.
+    :rtype: str
+    """
+    lowest_version = 'zebra'
+    for svc in current_versions.keys():
+        if current_versions[svc] < lowest_version:
+            lowest_version = current_versions[svc]
+    return lowest_version
+
+
+def determine_next_openstack_release(release):
+    """Determine the next release after the one passed as a str.
+
+    The returned value is a tuple of the form: ('2020.1', 'ussuri')
+
+    :param release: the release to use as the base
+    :type release: str
+    :returns: the release tuple immediately after the current one.
+    :rtype: Tuple[str, str]
+    :raises: KeyError if the current release doesn't actually exist
+    """
+    old_index = list(OPENSTACK_CODENAMES.values()).index(release)
+    new_index = old_index + 1
+    return list(OPENSTACK_CODENAMES.items())[new_index]
+
+
+def determine_new_source(ubuntu_version, current_source, new_release,
+                         single_increment=True):
+    """Determine the new source/openstack-origin value  based on new release.
+
+    This takes the ubuntu_version and the current_source (in the form of
+    'distro' or 'cloud:xenial-mitaka') and either converts it to a new source,
+    or returns None if the new_release will match the current_source (i.e. it's
+    already at the right release), or it's simply not possible.
+
+    If single_increment is set, then the returned source will only be returned
+    if the new_release is one more than the release in the current source.
+
+    :param ubuntu_version: the ubuntu version that the app is installed on.
+    :type ubuntu_version: str
+    :param current_source: a source in the form of 'distro' or
+        'cloud:xenial-mitaka'
+    :type current_source: str
+    :param new_release: a new OpenStack version codename. e.g. 'stein'
+    :type new_release: str
+    :param single_increment: If True, only allow single increment upgrade.
+    :type single_increment: boolean
+    :returns: The new source in the form of 'cloud:bionic-train' or None if not
+        possible
+    :rtype: Optional[str]
+    :raises: KeyError if any of the strings don't correspond to known values.
+    """
+    logging.warn("determine_new_source: locals: %s", locals())
+    if current_source == 'distro':
+        # convert to a ubuntu-openstack pair
+        current_source = "cloud:{}-{}".format(
+            ubuntu_version, UBUNTU_OPENSTACK_RELEASE[ubuntu_version])
+    # strip out the current openstack version
+    if ':' not in current_source:
+        current_source = "cloud:{}-{}".format(ubuntu_version, current_source)
+    pair = current_source.split(':')[1]
+    u_version, os_version = pair.split('-', 2)
+    if u_version != ubuntu_version:
+        logging.warn("determine_new_source: ubuntu_versions don't match: "
+                     "%s != %s" % (ubuntu_version, u_version))
+        return None
+    # determine versions
+    openstack_codenames = list(OPENSTACK_CODENAMES.values())
+    old_index = openstack_codenames.index(os_version)
+    try:
+        new_os_version = openstack_codenames[old_index + 1]
+    except IndexError:
+        logging.warn("determine_new_source: no OpenStack version after "
+                     "'%s'" % os_version)
+        return None
+    if single_increment and new_release != new_os_version:
+        logging.warn("determine_new_source: requested version '%s' not a "
+                     "single increment from '%s' which is '%s'" % (
+                         new_release, os_version, new_os_version))
+        return None
+    # now check that there is a combination of u_version-new_os_version
+    new_pair = "{}_{}".format(u_version, new_os_version)
+    if new_pair not in OPENSTACK_RELEASES_PAIRS:
+        logging.warn("determine_new_source: now release pair candidate for "
+                     " combination cloud:%s-%s" % (u_version, new_os_version))
+        return None
+    return "cloud:{}-{}".format(u_version, new_os_version)