From 282b8be577b9398c1af8415d0e521ee71a1ae719 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Wed, 6 Jan 2021 13:09:41 +0000 Subject: [PATCH] Update openstack upgrade tests for focal (#476) This patch modifies the existing openstack upgrade tests so that they work with focal (by explicitly supporting mysql-innodb-cluster), and are also interruptable and resumable (at a charm level). It also makes them work with the udpated 'get_upgrade_groups()' that ultimately gets a List of Tuples rather than a dictionary. --- .../test_zaza_utilities_openstack_upgrade.py | 118 +++++++++----- .../charm_tests/openstack_upgrade/__init__.py | 15 ++ .../charm_tests/openstack_upgrade/tests.py | 148 ++++++++++++++++++ zaza/openstack/charm_tests/vault/setup.py | 3 +- zaza/openstack/utilities/openstack.py | 33 ++-- zaza/openstack/utilities/openstack_upgrade.py | 136 +++++++++++----- zaza/openstack/utilities/upgrade_utils.py | 119 +++++++++++++- 7 files changed, 483 insertions(+), 89 deletions(-) create mode 100644 zaza/openstack/charm_tests/openstack_upgrade/__init__.py create mode 100644 zaza/openstack/charm_tests/openstack_upgrade/tests.py diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py index 4d619c5..3782926 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -44,25 +44,37 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.patch_object( openstack_upgrade.zaza.model, "get_application_config") + self.patch_object( + openstack_upgrade.zaza.model, + "block_until_all_units_idle") + self.patch_object( + openstack_upgrade, + "block_until_mysql_innodb_cluster_has_rw") def _get_application_config(app, model_name=None): app_config = { - 'ceph-mon': {'verbose': True, 'source': 'old-src'}, - 'neutron-openvswitch': {'verbose': True}, - 'ntp': {'verbose': True}, - 'percona-cluster': {'verbose': True, 'source': 'old-src'}, + 'ceph-mon': {'verbose': {'value': True}, + 'source': {'value': 'old-src'}}, + 'neutron-openvswitch': {'verbose': {'value': True}}, + 'ntp': {'verbose': {'value': True}}, + 'percona-cluster': {'verbose': {'value': True}, + 'source': {'value': 'old-src'}}, 'cinder': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, 'neutron-api': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, 'nova-compute': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, + 'mysql-innodb-cluster': { + 'verbose': {'value': True}, + 'source': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': True}}, } return app_config[app] self.get_application_config.side_effect = _get_application_config @@ -74,6 +86,10 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): 'subordinate-to': 'nova-compute'}, 'ntp': { # Filter as it has no source option 'charm': 'cs:ntp'}, + 'mysql-innodb-cluster': { + 'charm': 'cs:mysql-innodb-cluster', + 'units': { + 'mysql-innodb-cluster/0': {}}}, 'nova-compute': { 'charm': 'cs:nova-compute', 'units': { @@ -115,7 +131,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): model_name=None, raise_on_failure=True) - def test_action_upgrade_group(self): + def test_action_upgrade_apps(self): self.patch_object(openstack_upgrade, "pause_units") self.patch_object(openstack_upgrade, "action_unit_upgrade") self.patch_object(openstack_upgrade, "resume_units") @@ -127,7 +143,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): 'nova-compute': [mock_nova_compute_0], 'cinder': [mock_cinder_1]} self.get_units.side_effect = lambda app, model_name: units[app] - openstack_upgrade.action_upgrade_group(['nova-compute', 'cinder']) + openstack_upgrade.action_upgrade_apps(['nova-compute', 'cinder']) pause_calls = [ mock.call(['cinder-hacluster/0'], model_name=None), mock.call(['nova-compute/0', 'cinder/1'], model_name=None)] @@ -142,6 +158,30 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): mock.call(['cinder-hacluster/0'], model_name=None)] self.resume_units.assert_has_calls(resume_calls, any_order=False) + def test_action_upgrade_apps_mysql_innodb_cluster(self): + """Verify that mysql-innodb-cluster is settled before complete.""" + self.patch_object(openstack_upgrade, "pause_units") + self.patch_object(openstack_upgrade, "action_unit_upgrade") + self.patch_object(openstack_upgrade, "resume_units") + mock_mysql_innodb_cluster_0 = mock.MagicMock() + mock_mysql_innodb_cluster_0.entity_id = 'mysql-innodb-cluster/0' + units = {'mysql-innodb-cluster': [mock_mysql_innodb_cluster_0]} + self.get_units.side_effect = lambda app, model_name: units[app] + openstack_upgrade.action_upgrade_apps(['mysql-innodb-cluster']) + pause_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.pause_units.assert_has_calls(pause_calls, any_order=False) + action_unit_upgrade_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.action_unit_upgrade.assert_has_calls( + action_unit_upgrade_calls, + any_order=False) + resume_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.resume_units.assert_has_calls(resume_calls, any_order=False) + self.block_until_mysql_innodb_cluster_has_rw.assert_called_once_with( + None) + def test_set_upgrade_application_config(self): openstack_upgrade.set_upgrade_application_config( ['neutron-api', 'cinder'], @@ -177,17 +217,23 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.assertFalse( openstack_upgrade.is_action_upgradable('percona-cluster')) + def test_is_already_upgraded(self): + self.assertTrue( + openstack_upgrade.is_already_upgraded('cinder', 'old-src')) + self.assertFalse( + openstack_upgrade.is_already_upgraded('cinder', 'new-src')) + def test_run_action_upgrade(self): self.patch_object(openstack_upgrade, "set_upgrade_application_config") - self.patch_object(openstack_upgrade, "action_upgrade_group") - openstack_upgrade.run_action_upgrade( + self.patch_object(openstack_upgrade, "action_upgrade_apps") + openstack_upgrade.run_action_upgrades( ['cinder', 'neutron-api'], 'new-src') self.set_upgrade_application_config.assert_called_once_with( ['cinder', 'neutron-api'], 'new-src', model_name=None) - self.action_upgrade_group.assert_called_once_with( + self.action_upgrade_apps.assert_called_once_with( ['cinder', 'neutron-api'], model_name=None) @@ -196,7 +242,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.patch_object( openstack_upgrade.zaza.model, 'block_until_all_units_idle') - openstack_upgrade.run_all_in_one_upgrade( + openstack_upgrade.run_all_in_one_upgrades( ['percona-cluster'], 'new-src') self.set_upgrade_application_config.assert_called_once_with( @@ -207,34 +253,36 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.block_until_all_units_idle.assert_called_once_with() def test_run_upgrade(self): - self.patch_object(openstack_upgrade, "run_all_in_one_upgrade") - self.patch_object(openstack_upgrade, "run_action_upgrade") - openstack_upgrade.run_upgrade( + self.patch_object(openstack_upgrade, "run_all_in_one_upgrades") + self.patch_object(openstack_upgrade, "run_action_upgrades") + openstack_upgrade.run_upgrade_on_apps( ['cinder', 'neutron-api', 'ceph-mon'], 'new-src') - self.run_all_in_one_upgrade.assert_called_once_with( + self.run_all_in_one_upgrades.assert_called_once_with( ['ceph-mon'], 'new-src', model_name=None) - self.run_action_upgrade.assert_called_once_with( + self.run_action_upgrades.assert_called_once_with( ['cinder', 'neutron-api'], 'new-src', model_name=None) def test_run_upgrade_tests(self): - self.patch_object(openstack_upgrade, "run_upgrade") + self.patch_object(openstack_upgrade, "run_upgrade_on_apps") self.patch_object(openstack_upgrade, "get_upgrade_groups") - self.get_upgrade_groups.return_value = { - 'Compute': ['nova-compute'], - 'Control Plane': ['cinder', 'neutron-api'], - 'Core Identity': ['keystone'], - 'Storage': ['ceph-mon'], - 'sweep_up': ['designate']} + self.get_upgrade_groups.return_value = [ + ('Compute', ['nova-compute']), + ('Control Plane', ['cinder', 'neutron-api']), + ('Core Identity', ['keystone']), + ('Storage', ['ceph-mon']), + ('sweep_up', ['designate'])] openstack_upgrade.run_upgrade_tests('new-src', model_name=None) run_upgrade_calls = [ + mock.call(['nova-compute'], 'new-src', model_name=None), + mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None), mock.call(['keystone'], 'new-src', model_name=None), mock.call(['ceph-mon'], 'new-src', model_name=None), - mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None), - mock.call(['nova-compute'], 'new-src', model_name=None), - mock.call(['designate'], 'new-src', model_name=None)] - self.run_upgrade.assert_has_calls(run_upgrade_calls, any_order=False) + mock.call(['designate'], 'new-src', model_name=None), + ] + self.run_upgrade_on_apps.assert_has_calls( + run_upgrade_calls, any_order=False) diff --git a/zaza/openstack/charm_tests/openstack_upgrade/__init__.py b/zaza/openstack/charm_tests/openstack_upgrade/__init__.py new file mode 100644 index 0000000..9a1ca53 --- /dev/null +++ b/zaza/openstack/charm_tests/openstack_upgrade/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for testing openstack upgrades.""" diff --git a/zaza/openstack/charm_tests/openstack_upgrade/tests.py b/zaza/openstack/charm_tests/openstack_upgrade/tests.py new file mode 100644 index 0000000..82c7fbb --- /dev/null +++ b/zaza/openstack/charm_tests/openstack_upgrade/tests.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define class for OpenStack Upgrade.""" + +import logging +import unittest + +from zaza.openstack.utilities import ( + cli as cli_utils, + upgrade_utils as upgrade_utils, + openstack as openstack_utils, + openstack_upgrade as openstack_upgrade, +) +from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest + + +class OpenStackUpgradeVMLaunchBase(object): + """A base class to peform a simple validation on the cloud. + + This wraps an OpenStack upgrade with a VM launch before and after the + upgrade. + + This test requires a full OpenStack including at least: keystone, glance, + nova-cloud-controller, nova-compute, neutron-gateway, neutron-api and + neutron-openvswitch. + + This class should be used as a base class to the upgrade 'test'. + """ + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenStackUpgradeMixin setUpClass") + super().setUpClass() + cls.lts = LTSGuestCreateTest() + cls.lts.setUpClass() + + def test_100_validate_pre_openstack_upgrade_cloud(self): + """Validate pre openstack upgrade.""" + logging.info("Validate pre-openstack-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + def test_500_validate_openstack_upgraded_cloud(self): + """Validate post openstack upgrade.""" + logging.info("Validate post-openstack-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + +class WaitForMySQL(unittest.TestCase): + """Helper test to wait on mysql-innodb-cluster to be fully ready. + + In practice this means that there is at least on R/W unit available. + Sometimes, after restarting units in the mysql-innodb-cluster, all the + units are R/O until the cluster picks the R/W unit. + """ + + @classmethod + def setUpClass(cls): + """Set up class.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_100_wait_for_happy_mysql_innodb_cluster(self): + """Wait for mysql cluster to have at least one R/W node.""" + logging.info("Starting wait for an R/W unit.") + openstack_upgrade.block_until_mysql_innodb_cluster_has_rw() + logging.info("Done .. all seems well.") + + +class OpenStackUpgradeTestsFocalUssuri(OpenStackUpgradeVMLaunchBase): + """Upgrade OpenStack from distro -> cloud:focal-victoria.""" + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_200_run_openstack_upgrade(self): + """Run openstack upgrade, but work out what to do.""" + openstack_upgrade.run_upgrade_tests("cloud:focal-victoria") + + +class OpenStackUpgradeTests(OpenStackUpgradeVMLaunchBase): + """A Principal Class to encapsulate OpenStack Upgrade Tests. + + A generic Test class that can discover which Ubuntu version and OpenStack + version to upgrade from. + + TODO: Not used at present. Use the declarative tests directly that choose + the version to upgrade to. The functions that this class depends on need a + bit more work regarding how the determine which version to go to. + """ + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_200_run_openstack_upgrade(self): + """Run openstack upgrade, but work out what to do. + + TODO: This is really inefficient at the moment, and doesn't (yet) + determine which ubuntu version to work from. Don't use until we can + make it better. + """ + # TODO: work out the most recent Ubuntu version; we assume this is the + # version that OpenStack is running on. + ubuntu_version = "focal" + logging.info("Getting all principle applications ...") + principle_services = upgrade_utils.get_all_principal_applications() + logging.info( + "Getting OpenStack vesions from principal applications ...") + current_versions = openstack_utils.get_current_os_versions( + principle_services) + logging.info("current versions: %s" % current_versions) + # Find the lowest value openstack release across all services and make + # sure all servcies are upgraded to one release higher than the lowest + from_version = upgrade_utils.get_lowest_openstack_version( + current_versions) + logging.info("from version: %s" % from_version) + to_version = upgrade_utils.determine_next_openstack_release( + from_version)[1] + logging.info("to version: %s" % to_version) + # TODO: need to determine the ubuntu base verion that is being upgraded + target_source = upgrade_utils.determine_new_source( + ubuntu_version, from_version, to_version, single_increment=True) + logging.info("target source: %s" % target_source) + assert target_source is not None + openstack_upgrade.run_upgrade_tests(target_source) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index adfb92d..1e4e05e 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -174,7 +174,8 @@ def auto_initialize(cacert=None, validation_application='keystone', wait=True): zaza.model.wait_for_agent_status() test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) + states=test_config.get('target_deploy_status', {}), + timeout=7200) if validation_application: validate_ca(cacertificate, application=validation_application) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index a93e61d..3dd5361 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -16,6 +16,23 @@ This module contains a number of functions for interacting with OpenStack. """ +import datetime +import io +import itertools +import juju_wait +import logging +import os +import paramiko +import re +import six +import subprocess +import sys +import tempfile +import tenacity +import textwrap +import urllib + + from .os_versions import ( OPENSTACK_CODENAMES, SWIFT_CODENAMES, @@ -48,21 +65,6 @@ from neutronclient.common import exceptions as neutronexceptions from octaviaclient.api.v2 import octavia as octaviaclient from swiftclient import client as swiftclient -import datetime -import io -import itertools -import juju_wait -import logging -import os -import paramiko -import re -import six -import subprocess -import sys -import tempfile -import tenacity -import textwrap -import urllib import zaza @@ -1554,6 +1556,7 @@ def get_current_os_versions(deployed_applications, model_name=None): for application in UPGRADE_SERVICES: if application['name'] not in deployed_applications: continue + logging.info("looking at application: {}".format(application)) version = generic_utils.get_pkg_version(application['name'], application['type']['pkg'], diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 9ad4d50..c279e58 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -95,8 +95,8 @@ async def async_action_unit_upgrade(units, model_name=None): action_unit_upgrade = sync_wrapper(async_action_unit_upgrade) -def action_upgrade_group(applications, model_name=None): - """Upgrade units using action managed upgrades. +def action_upgrade_apps(applications, model_name=None): + """Upgrade units in the applications using action managed upgrades. Upgrade all units of the given applications using action managed upgrades. This involves the following process: @@ -142,6 +142,45 @@ def action_upgrade_group(applications, model_name=None): done.extend(target) + # Ensure that mysql-innodb-cluster has at least one R/W group (it can get + # into a state where all are R/O whilst it is sorting itself out after an + # openstack_upgrade + if "mysql-innodb-cluster" in applications: + block_until_mysql_innodb_cluster_has_rw(model_name) + + # Now we need to wait for the model to go back to idle. + zaza.model.block_until_all_units_idle(model_name) + + +async def async_block_until_mysql_innodb_cluster_has_rw(model=None, + timeout=None): + """Block until the mysql-innodb-cluster is in a healthy state. + + Curiously, after a series of pauses and restarts (e.g. during an upgrade) + the mysql-innodb-cluster charms may not yet have agreed which one is the + R/W node; i.e. they are all R/O. Anyway, eventually they sort it out and + one jumps to the front and says "it's me!". This is detected, externally, + by the status line including R/W in the output. + + This function blocks until that happens so that no charm attempts to have a + chat with the mysql server before it has settled, thus breaking the whole + test. + """ + async def async_check_workload_messages_for_rw(model=None): + """Return True if a least one work message contains R/W.""" + status = await zaza.model.async_get_status() + app_status = status.applications.get("mysql-innodb-cluster") + units_data = app_status.units.values() + workload_statuses = [d.workload_status.info for d in units_data] + return any("R/W" in s for s in workload_statuses) + + await zaza.model.async_block_until(async_check_workload_messages_for_rw, + timeout=timeout) + + +block_until_mysql_innodb_cluster_has_rw = sync_wrapper( + async_block_until_mysql_innodb_cluster_has_rw) + def set_upgrade_application_config(applications, new_source, action_managed=True, model_name=None): @@ -150,7 +189,7 @@ def set_upgrade_application_config(applications, new_source, Set the charm config for upgrade. :param applications: List of application names. - :type applications: [] + :type applications: List[str] :param new_source: New package origin. :type new_source: str :param action_managed: Whether to set action-managed-upgrade config option. @@ -180,8 +219,8 @@ def set_upgrade_application_config(applications, new_source, def is_action_upgradable(app, model_name=None): """Can application be upgraded using action managed upgrade method. - :param new_source: New package origin. - :type new_source: str + :param app: The application to check + :type app: str :param model_name: Name of model to query. :type model_name: str :returns: Whether app be upgraded using action managed upgrade method. @@ -196,66 +235,95 @@ def is_action_upgradable(app, model_name=None): return supported -def run_action_upgrade(group, new_source, model_name=None): +def is_already_upgraded(app, new_src, model_name=None): + """Return True if the app has already been upgraded. + + :param app: The application to check + :type app: str + :param new_src: the new source (distro, cloud:x-y, etc.) + :type new_src: str + :param model_name: Name of model to query. + :type model_name: str + :returns: Whether app be upgraded using action managed upgrade method. + :rtype: bool + """ + config = zaza.model.get_application_config(app, model_name=model_name) + try: + src = config['openstack-origin']['value'] + key_was = 'openstack-origin' + except KeyError: + src = config['source']['value'] + key_was = 'source' + logging.info("origin for {} is {}={}".format(app, key_was, src)) + return src == new_src + + +def run_action_upgrades(apps, new_source, model_name=None): """Upgrade payload of all applications in group using action upgrades. - :param group: List of applications to upgrade. - :type group + :param apps: List of applications to upgrade. + :type apps: List[str] :param new_source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ - set_upgrade_application_config(group, new_source, model_name=model_name) - action_upgrade_group(group, model_name=model_name) + set_upgrade_application_config(apps, new_source, model_name=model_name) + action_upgrade_apps(apps, model_name=model_name) -def run_all_in_one_upgrade(group, new_source, model_name=None): +def run_all_in_one_upgrades(apps, new_source, model_name=None): """Upgrade payload of all applications in group using all-in-one method. - :param group: List of applications to upgrade. - :type group: [] + :param apps: List of applications to upgrade. + :type apps: List[str] :source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ set_upgrade_application_config( - group, + apps, new_source, model_name=model_name, action_managed=False) zaza.model.block_until_all_units_idle() -def run_upgrade(group, new_source, model_name=None): +def run_upgrade_on_apps(apps, new_source, model_name=None): """Upgrade payload of all applications in group. Upgrade apps using action managed upgrades where possible and fallback to all_in_one method. - :param group: List of applications to upgrade. - :type group: [] + :param apps: List of applications to upgrade. + :type apps: [] :param new_source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ - action_upgrade = [] - all_in_one_upgrade = [] - for app in group: + action_upgrades = [] + all_in_one_upgrades = [] + for app in apps: + if is_already_upgraded(app, new_source, model_name=model_name): + logging.info("Application '%s' is already upgraded. Skipping.", + app) + continue if is_action_upgradable(app, model_name=model_name): - action_upgrade.append(app) + action_upgrades.append(app) else: - all_in_one_upgrade.append(app) - run_all_in_one_upgrade( - all_in_one_upgrade, - new_source, - model_name=model_name) - run_action_upgrade( - action_upgrade, - new_source, - model_name=model_name) + all_in_one_upgrades.append(app) + if all_in_one_upgrades: + run_all_in_one_upgrades( + all_in_one_upgrades, + new_source, + model_name=model_name) + if action_upgrades: + run_action_upgrades( + action_upgrades, + new_source, + model_name=model_name) def run_upgrade_tests(new_source, model_name=None): @@ -270,8 +338,6 @@ def run_upgrade_tests(new_source, model_name=None): :type model_name: str """ groups = get_upgrade_groups(model_name=model_name) - run_upgrade(groups['Core Identity'], new_source, model_name=model_name) - run_upgrade(groups['Storage'], new_source, model_name=model_name) - run_upgrade(groups['Control Plane'], new_source, model_name=model_name) - run_upgrade(groups['Compute'], new_source, model_name=model_name) - run_upgrade(groups['sweep_up'], new_source, model_name=model_name) + for name, apps in groups: + logging.info("Performing upgrade of %s", name) + run_upgrade_on_apps(apps, new_source, model_name=model_name) diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 995e0bb..8e3a063 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -19,6 +19,12 @@ import logging import re import zaza.model +import zaza.utilities.juju +from zaza.openstack.utilities.os_versions import ( + OPENSTACK_CODENAMES, + UBUNTU_OPENSTACK_RELEASE, + OPENSTACK_RELEASES_PAIRS, +) SERVICE_GROUPS = ( @@ -46,7 +52,7 @@ def get_upgrade_candidates(model_name=None, filters=None): :param filters: List of filter functions to apply :type filters: List[fn] :returns: List of application that can have their payload upgraded. - :rtype: [] + :rtype: Dict[str, Dict[str, ANY]] """ if filters is None: filters = [] @@ -163,8 +169,8 @@ def get_series_upgrade_groups(model_name=None, extra_filters=None): :param model_name: Name of model to query. :type model_name: str - :returns: Dict of group lists keyed on group name. - :rtype: collections.OrderedDict + :returns: List of tuples(group name, applications) + :rtype: List[Tuple[str, Dict[str, ANY]]] """ filters = [_filter_subordinates] filters = _apply_extra_filters(filters, extra_filters) @@ -226,3 +232,110 @@ def extract_charm_name_from_url(charm_url): """ charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) return charm_name.split(':')[-1] + + +def get_all_principal_applications(model_name=None): + """Return a list of all the prinical applications in the model. + + :param model_name: Optional model name + :type model_name: Optional[str] + :returns: List of principal application names + :rtype: List[str] + """ + status = zaza.utilities.juju.get_full_juju_status(model_name=model_name) + return [application for application in status.applications.keys() + if not status.applications.get(application)['subordinate-to']] + + +def get_lowest_openstack_version(current_versions): + """Get the lowest OpenStack version from the list of current versions. + + :param current_versions: The list of versions + :type current_versions: List[str] + :returns: the lowest version currently installed. + :rtype: str + """ + lowest_version = 'zebra' + for svc in current_versions.keys(): + if current_versions[svc] < lowest_version: + lowest_version = current_versions[svc] + return lowest_version + + +def determine_next_openstack_release(release): + """Determine the next release after the one passed as a str. + + The returned value is a tuple of the form: ('2020.1', 'ussuri') + + :param release: the release to use as the base + :type release: str + :returns: the release tuple immediately after the current one. + :rtype: Tuple[str, str] + :raises: KeyError if the current release doesn't actually exist + """ + old_index = list(OPENSTACK_CODENAMES.values()).index(release) + new_index = old_index + 1 + return list(OPENSTACK_CODENAMES.items())[new_index] + + +def determine_new_source(ubuntu_version, current_source, new_release, + single_increment=True): + """Determine the new source/openstack-origin value based on new release. + + This takes the ubuntu_version and the current_source (in the form of + 'distro' or 'cloud:xenial-mitaka') and either converts it to a new source, + or returns None if the new_release will match the current_source (i.e. it's + already at the right release), or it's simply not possible. + + If single_increment is set, then the returned source will only be returned + if the new_release is one more than the release in the current source. + + :param ubuntu_version: the ubuntu version that the app is installed on. + :type ubuntu_version: str + :param current_source: a source in the form of 'distro' or + 'cloud:xenial-mitaka' + :type current_source: str + :param new_release: a new OpenStack version codename. e.g. 'stein' + :type new_release: str + :param single_increment: If True, only allow single increment upgrade. + :type single_increment: boolean + :returns: The new source in the form of 'cloud:bionic-train' or None if not + possible + :rtype: Optional[str] + :raises: KeyError if any of the strings don't correspond to known values. + """ + logging.warn("determine_new_source: locals: %s", locals()) + if current_source == 'distro': + # convert to a ubuntu-openstack pair + current_source = "cloud:{}-{}".format( + ubuntu_version, UBUNTU_OPENSTACK_RELEASE[ubuntu_version]) + # strip out the current openstack version + if ':' not in current_source: + current_source = "cloud:{}-{}".format(ubuntu_version, current_source) + pair = current_source.split(':')[1] + u_version, os_version = pair.split('-', 2) + if u_version != ubuntu_version: + logging.warn("determine_new_source: ubuntu_versions don't match: " + "%s != %s" % (ubuntu_version, u_version)) + return None + # determine versions + openstack_codenames = list(OPENSTACK_CODENAMES.values()) + old_index = openstack_codenames.index(os_version) + try: + new_os_version = openstack_codenames[old_index + 1] + except IndexError: + logging.warn("determine_new_source: no OpenStack version after " + "'%s'" % os_version) + return None + if single_increment and new_release != new_os_version: + logging.warn("determine_new_source: requested version '%s' not a " + "single increment from '%s' which is '%s'" % ( + new_release, os_version, new_os_version)) + return None + # now check that there is a combination of u_version-new_os_version + new_pair = "{}_{}".format(u_version, new_os_version) + if new_pair not in OPENSTACK_RELEASES_PAIRS: + logging.warn("determine_new_source: now release pair candidate for " + " combination cloud:%s-%s" % (u_version, new_os_version)) + return None + return "cloud:{}-{}".format(u_version, new_os_version)