diff --git a/requirements.txt b/requirements.txt index 68f8f71..cbf31df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ aiounittest -asyncio async_generator boto3 juju diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py index f711bb2..bed43b3 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -172,38 +172,6 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): {'source': 'new-src'}, model_name=None) - def test__extract_charm_name_from_url(self): - self.assertEqual( - openstack_upgrade._extract_charm_name_from_url( - 'local:bionic/heat-12'), - 'heat') - self.assertEqual( - openstack_upgrade._extract_charm_name_from_url( - 'cs:bionic/heat-12'), - 'heat') - self.assertEqual( - openstack_upgrade._extract_charm_name_from_url('cs:heat'), - 'heat') - - def test_get_upgrade_candidates(self): - expect = copy.deepcopy(self.juju_status.applications) - del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST - del expect['ntp'] # Filter as it has no source option - del expect['neutron-openvswitch'] # Filter as it is a subordinates - self.assertEqual( - openstack_upgrade.get_upgrade_candidates(), - expect) - - def test_get_upgrade_groups(self): - self.assertEqual( - openstack_upgrade.get_upgrade_groups(), - { - 'Compute': ['nova-compute'], - 'Control Plane': ['cinder'], - 'Core Identity': [], - 'Storage': [], - 'sweep_up': []}) - def test_is_action_upgradable(self): self.assertTrue( openstack_upgrade.is_action_upgradable('cinder')) diff --git a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py new file mode 100644 index 0000000..e1d5ff8 --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py @@ -0,0 +1,112 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import mock + +import unit_tests.utils as ut_utils +import zaza.openstack.utilities.upgrade_utils as openstack_upgrade + + +class TestUpgradeUtils(ut_utils.BaseTestCase): + def setUp(self): + super(TestUpgradeUtils, self).setUp() + self.patch_object( + openstack_upgrade.zaza.model, + "get_units") + self.juju_status = mock.MagicMock() + self.patch_object( + openstack_upgrade.zaza.model, + "get_status", + return_value=self.juju_status) + self.patch_object( + openstack_upgrade.zaza.model, + "get_application_config") + + def _get_application_config(app, model_name=None): + app_config = { + 'ceph-mon': {'verbose': True, 'source': 'old-src'}, + 'neutron-openvswitch': {'verbose': True}, + 'ntp': {'verbose': True}, + 'percona-cluster': {'verbose': True, 'source': 'old-src'}, + 'cinder': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + 'neutron-api': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + 'nova-compute': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + } + return app_config[app] + self.get_application_config.side_effect = _get_application_config + self.juju_status.applications = { + 'mydb': { # Filter as it is on UPGRADE_EXCLUDE_LIST + 'charm': 'cs:percona-cluster'}, + 'neutron-openvswitch': { # Filter as it is a subordinates + 'charm': 'cs:neutron-openvswitch', + 'subordinate-to': 'nova-compute'}, + 'ntp': { # Filter as it has no source option + 'charm': 'cs:ntp'}, + 'nova-compute': { + 'charm': 'cs:nova-compute', + 'units': { + 'nova-compute/0': { + 'subordinates': { + 'neutron-openvswitch/2': { + 'charm': 'cs:neutron-openvswitch-22'}}}}}, + 'cinder': { + 'charm': 'cs:cinder-23', + 'units': { + 'cinder/1': { + 'subordinates': { + 'cinder-hacluster/0': { + 'charm': 'cs:hacluster-42'}, + 'cinder-ceph/3': { + 'charm': 'cs:cinder-ceph-2'}}}}}} + + def test_get_upgrade_candidates(self): + expect = copy.deepcopy(self.juju_status.applications) + del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST + del expect['ntp'] # Filter as it has no source option + del expect['neutron-openvswitch'] # Filter as it is a subordinates + self.assertEqual( + openstack_upgrade.get_upgrade_candidates(), + expect) + + def test_get_upgrade_groups(self): + self.assertEqual( + openstack_upgrade.get_upgrade_groups(), + { + 'Core Identity': [], + 'Control Plane': ['cinder'], + 'Data Plane': ['nova-compute'], + 'sweep_up': []}) + + def test_extract_charm_name_from_url(self): + self.assertEqual( + openstack_upgrade.extract_charm_name_from_url( + 'local:bionic/heat-12'), + 'heat') + self.assertEqual( + openstack_upgrade.extract_charm_name_from_url( + 'cs:bionic/heat-12'), + 'heat') + self.assertEqual( + openstack_upgrade.extract_charm_name_from_url('cs:heat'), + 'heat') diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index f94e93d..6f6eb4e 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -16,6 +16,7 @@ """Define class for Series Upgrade.""" +import asyncio import logging import os import unittest @@ -24,6 +25,7 @@ from zaza import model from zaza.openstack.utilities import ( cli as cli_utils, series_upgrade as series_upgrade_utils, + upgrade_utils as upgrade_utils, ) from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest @@ -54,45 +56,45 @@ class SeriesUpgradeTest(unittest.TestCase): pause_non_leader_primary = True post_upgrade_functions = [] # Skip subordinates - if applications[application]["subordinate-to"]: + if app_details["subordinate-to"]: continue - if "easyrsa" in applications[application]["charm"]: + if "easyrsa" in app_details["charm"]: logging.warn("Skipping series upgrade of easyrsa Bug #1850121") continue - if "etcd" in applications[application]["charm"]: + if "etcd" in app_details["charm"]: logging.warn("Skipping series upgrade of easyrsa Bug #1850124") continue - if "percona-cluster" in applications[application]["charm"]: + if "percona-cluster" in app_details["charm"]: origin = "source" pause_non_leader_primary = True pause_non_leader_subordinate = True - if "rabbitmq-server" in applications[application]["charm"]: + if "rabbitmq-server" in app_details["charm"]: origin = "source" pause_non_leader_primary = True pause_non_leader_subordinate = False - if "nova-compute" in applications[application]["charm"]: + if "nova-compute" in app_details["charm"]: pause_non_leader_primary = False pause_non_leader_subordinate = False - if "ceph" in applications[application]["charm"]: + if "ceph" in app_details["charm"]: origin = "source" pause_non_leader_primary = False pause_non_leader_subordinate = False - if "designate-bind" in applications[application]["charm"]: + if "designate-bind" in app_details["charm"]: origin = None - if "tempest" in applications[application]["charm"]: + if "tempest" in app_details["charm"]: origin = None - if "memcached" in applications[application]["charm"]: + if "memcached" in app_details["charm"]: origin = None pause_non_leader_primary = False pause_non_leader_subordinate = False - if "vault" in applications[application]["charm"]: + if "vault" in app_details["charm"]: origin = None pause_non_leader_primary = False pause_non_leader_subordinate = True post_upgrade_functions = [ ('zaza.openstack.charm_tests.vault.setup.' 'mojo_unseal_by_unit')] - if "mongodb" in applications[application]["charm"]: + if "mongodb" in app_details["charm"]: # Mongodb needs to run series upgrade # on its secondaries first. series_upgrade_utils.series_upgrade_non_leaders_first( @@ -117,7 +119,7 @@ class SeriesUpgradeTest(unittest.TestCase): files=self.files, post_upgrade_functions=post_upgrade_functions) - if "rabbitmq-server" in applications[application]["charm"]: + if "rabbitmq-server" in app_details["charm"]: logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( @@ -126,7 +128,7 @@ class SeriesUpgradeTest(unittest.TestCase): action_params={}) model.block_until_all_units_idle() - if "percona-cluster" in applications[application]["charm"]: + if "percona-cluster" in app_details["charm"]: logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( @@ -215,5 +217,165 @@ class XenialBionicSeriesUpgrade(SeriesUpgradeTest): cls.to_series = "bionic" +class ParallelSeriesUpgradeTest(unittest.TestCase): + """Class to encapsulate Sereis Upgrade Tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + cli_utils.setup_logging() + cls.from_series = None + cls.to_series = None + cls.workaround_script = None + cls.files = [] + + def test_200_run_series_upgrade(self): + """Run series upgrade.""" + # Set Feature Flag + os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + upgrade_groups = upgrade_utils.get_upgrade_groups() + applications = model.get_status().applications + upgrade_groups['support'] = [ + app for app in upgrade_utils.UPGRADE_EXCLUDE_LIST + if app in applications.keys()] + upgrade_groups['deferred'] = [] + completed_machines = [] + deferred_applications = [] + for group_name, group in upgrade_groups.items(): + logging.warn("About to upgrade {} ({})".format(group_name, group)) + upgrade_group = [] + for application, app_details in applications.items(): + # Defaults + origin = "openstack-origin" + pause_non_leader_subordinate = True + pause_non_leader_primary = True + post_upgrade_functions = [] + name = upgrade_utils.extract_charm_name_from_url( + app_details['charm']) + if name not in group and application not in group: + if group_name is not "deferred" and \ + name not in upgrade_groups['deferred']: + upgrade_groups['deferred'].append(name) + continue + if group_name is not "deferred" and \ + name in upgrade_groups['deferred']: + upgrade_groups['deferred'].remove(name) + # Skip subordinates + if app_details["subordinate-to"]: + continue + if "easyrsa" in app_details["charm"]: + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850121") + continue + if "etcd" in app_details["charm"]: + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850124") + continue + logging.warn("About to upgrade {}".format(application)) + if "percona-cluster" in app_details["charm"]: + origin = "source" + pause_non_leader_primary = True + pause_non_leader_subordinate = True + if "rabbitmq-server" in app_details["charm"]: + origin = "source" + pause_non_leader_primary = True + pause_non_leader_subordinate = False + if "nova-compute" in app_details["charm"]: + pause_non_leader_primary = False + pause_non_leader_subordinate = False + if "ceph" in app_details["charm"]: + origin = "source" + pause_non_leader_primary = False + pause_non_leader_subordinate = False + if "designate-bind" in app_details["charm"]: + origin = None + if "tempest" in app_details["charm"]: + origin = None + if "memcached" in app_details["charm"]: + origin = None + pause_non_leader_primary = False + pause_non_leader_subordinate = False + if "vault" in app_details["charm"]: + origin = None + pause_non_leader_primary = False + pause_non_leader_subordinate = True + post_upgrade_functions = [ + ('zaza.openstack.charm_tests.vault.setup.' + 'mojo_unseal_by_unit')] + if "mongodb" in app_details["charm"]: + # Mongodb needs to run series upgrade + # on its secondaries first. + upgrade_group.append(series_upgrade_utils.async_series_upgrade_non_leaders_first( + application, + from_series=self.from_series, + to_series=self.to_series, + completed_machines=completed_machines, + post_upgrade_functions=post_upgrade_functions)) + continue + + # The rest are likley APIs use defaults + + upgrade_group.append(series_upgrade_utils.async_series_upgrade_application( + application, + pause_non_leader_primary=pause_non_leader_primary, + pause_non_leader_subordinate=pause_non_leader_subordinate, + from_series=self.from_series, + to_series=self.to_series, + origin=origin, + completed_machines=completed_machines, + workaround_script=self.workaround_script, + files=self.files, + post_upgrade_functions=post_upgrade_functions)) + + asyncio.get_event_loop().run_until_complete( + asyncio.gather(*upgrade_group, return_exceptions=True)) + if "rabbitmq-server" in group: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) + model.block_until_all_units_idle() + + if "percona-cluster" in group: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) + model.block_until_all_units_idle() + + + +class ParallelTrustyXenialSeriesUpgrade(ParallelSeriesUpgradeTest): + """Trusty to Xenial Series Upgrade. + + Makes no assumptions about what is in the deployment. + """ + + @classmethod + def setUpClass(cls): + """Run setup for Trusty to Xenial Series Upgrades.""" + super(ParallelTrustyXenialSeriesUpgrade, cls).setUpClass() + cls.from_series = "trusty" + cls.to_series = "xenial" + + +class ParallelXenialBionicSeriesUpgrade(ParallelSeriesUpgradeTest): + """Xenial to Bionic Series Upgrade. + + Makes no assumptions about what is in the deployment. + """ + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(ParallelXenialBionicSeriesUpgrade, cls).setUpClass() + cls.from_series = "xenial" + cls.to_series = "bionic" + if __name__ == "__main__": unittest.main() + diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 065d8ce..8e85f60 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -14,6 +14,7 @@ """Collection of functions that did not fit anywhere else.""" +import asyncio import logging import os import socket @@ -205,6 +206,25 @@ def set_origin(application, origin='openstack-origin', pocket='distro'): model.set_application_config(application, {origin: pocket}) +async def async_set_origin(application, origin='openstack-origin', + pocket='distro'): + """Set the configuration option for origin source. + + :param application: Name of application to upgrade series + :type application: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param pocket: Origin source cloud pocket. + i.e. 'distro' or 'cloud:xenial-newton' + :type pocket: str + :returns: None + :rtype: None + """ + logging.info("Set origin on {} to {}".format(application, origin)) + await model.async_set_application_config(application, {origin: pocket}) + + def run_via_ssh(unit_name, cmd): """Run command on unit via ssh. @@ -227,6 +247,28 @@ def run_via_ssh(unit_name, cmd): logging.warn(e) +async def async_run_via_ssh(unit_name, cmd): + """Run command on unit via ssh. + + For executing commands on units when the juju agent is down. + + :param unit_name: Unit Name + :param cmd: Command to execute on remote unit + :type cmd: str + :returns: None + :rtype: None + """ + if "sudo" not in cmd: + # cmd.insert(0, "sudo") + cmd = "sudo {}".format(cmd) + cmd = ['juju', 'ssh', unit_name, cmd] + try: + await check_call(cmd) + except subprocess.CalledProcessError as e: + logging.warn("Failed command {} on {}".format(cmd, unit_name)) + logging.warn(e) + + def check_commands_on_units(commands, units): """Check that all commands in a list exit zero on all units in a list. @@ -270,6 +312,29 @@ def reboot(unit_name): pass +async def async_reboot(unit_name): + """Reboot unit. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + # NOTE: When used with series upgrade the agent will be down. + # Even juju run will not work + await async_run_via_ssh(unit_name, "sudo reboot && exit") + + +async def check_call(cmd): + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, cmd) + + def set_dpkg_non_interactive_on_unit( unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"): """Set dpkg options on unit. @@ -286,6 +351,22 @@ def set_dpkg_non_interactive_on_unit( model.run_on_unit(unit_name, cmd) +async def async_set_dpkg_non_interactive_on_unit( + unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"): + """Set dpkg options on unit. + + :param unit_name: Unit Name + :type unit_name: str + :param apt_conf_d: Apt.conf file to update + :type apt_conf_d: str + """ + DPKG_NON_INTERACTIVE = 'DPkg::options { "--force-confdef"; };' + # Check if the option exists. If not, add it to the apt.conf.d file + cmd = ("grep '{option}' {file_name} || echo '{option}' >> {file_name}" + .format(option=DPKG_NON_INTERACTIVE, file_name=apt_conf_d)) + await model.async_run_on_unit(unit_name, cmd) + + def get_process_id_list(unit_name, process_name, expect_success=True): """Get a list of process ID(s). diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 5d5c729..87a8159 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -16,7 +16,6 @@ This module contains a number of functions for upgrading OpenStack. """ -import re import logging import zaza.openstack.utilities.juju as juju_utils @@ -25,6 +24,8 @@ from zaza import sync_wrapper from zaza.openstack.utilities.upgrade_utils import ( SERVICE_GROUPS, UPGRADE_EXCLUDE_LIST, + get_upgrade_candidates, + get_upgrade_groups, ) @@ -179,88 +180,6 @@ def set_upgrade_application_config(applications, new_source, model_name=model_name) -def _extract_charm_name_from_url(charm_url): - """Extract the charm name from the charm url. - - E.g. Extract 'heat' from local:bionic/heat-12 - - :param charm_url: Name of model to query. - :type charm_url: str - :returns: Charm name - :rtype: str - """ - charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) - return charm_name.split(':')[-1] - - -def get_upgrade_candidates(model_name=None): - """Extract list of apps from model that can be upgraded. - - :param model_name: Name of model to query. - :type model_name: str - :returns: List of application that can have their payload upgraded. - :rtype: [] - """ - status = zaza.model.get_status(model_name=model_name) - candidates = {} - for app, app_config in status.applications.items(): - # Filter out subordinates - if app_config.get("subordinate-to"): - logging.warning( - "Excluding {} from upgrade, it is a subordinate".format(app)) - continue - - # Filter out charms on the naughty list - charm_name = _extract_charm_name_from_url(app_config['charm']) - if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: - logging.warning( - "Excluding {} from upgrade, on the exclude list".format(app)) - continue - - # Filter out charms that have no source option - charm_options = zaza.model.get_application_config( - app, model_name=model_name).keys() - src_options = ['openstack-origin', 'source'] - if not [x for x in src_options if x in charm_options]: - logging.warning( - "Excluding {} from upgrade, no src option".format(app)) - continue - - candidates[app] = app_config - return candidates - - -def get_upgrade_groups(model_name=None): - """Place apps in the model into their upgrade groups. - - Place apps in the model into their upgrade groups. If an app is deployed - but is not in SERVICE_GROUPS then it is placed in a sweep_up group. - - :param model_name: Name of model to query. - :type model_name: str - :returns: Dict of group lists keyed on group name. - :rtype: {} - """ - apps_in_model = get_upgrade_candidates(model_name=model_name) - - groups = {} - for phase_name, charms in SERVICE_GROUPS.items(): - group = [] - for app, app_config in apps_in_model.items(): - charm_name = _extract_charm_name_from_url(app_config['charm']) - if charm_name in charms: - group.append(app) - groups[phase_name] = group - - sweep_up = [] - for app in apps_in_model: - if not (app in [a for group in groups.values() for a in group]): - sweep_up.append(app) - - groups['sweep_up'] = sweep_up - return groups - - def is_action_upgradable(app, model_name=None): """Can application be upgraded using action managed upgrade method. diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index e495c64..bdd8491 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -101,6 +101,68 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", model.block_until_all_units_idle() +async def async_series_upgrade_non_leaders_first(application, from_series="trusty", + to_series="xenial", + completed_machines=[], + post_upgrade_functions=None): + """Series upgrade non leaders first. + + Wrap all the functionality to handle series upgrade for charms + which must have non leaders upgraded first. + + :param application: Name of application to upgrade series + :type application: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :returns: None + :rtype: None + """ + status = (await model.async_get_status()).applications[application] + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Series upgrade the non-leaders first + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + await async_series_upgrade(unit, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + await async_run_post_upgrade_functions(post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + .format(unit, machine, application)) + await model.async_block_until_all_units_idle() + + # Series upgrade the leader + machine = status["units"][leader]["machine"] + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + await async_series_upgrade(leader, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + .format(unit, machine, application)) + await model.async_block_until_all_units_idle() + + def series_upgrade_application(application, pause_non_leader_primary=True, pause_non_leader_subordinate=True, from_series="trusty", to_series="xenial", @@ -209,6 +271,144 @@ def series_upgrade_application(application, pause_non_leader_primary=True, model.block_until_all_units_idle() +async def async_series_upgrade_application(application, + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + from_series="trusty", + to_series="xenial", + origin='openstack-origin', + completed_machines=None, + files=None, workaround_script=None, + post_upgrade_functions=None): + """Series upgrade application. + + Wrap all the functionality to handle series upgrade for a given + application. Including pausing non-leader units. + + :param application: Name of application to upgrade series + :type application: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + if completed_machines is None: + completed_machines = [] + status = (await model.async_get_status()).applications[application] + + # For some applications (percona-cluster) the leader unit must upgrade + # first. For API applications the non-leader haclusters must be paused + # before upgrade. Finally, for some applications this is arbitrary but + # generalized. + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Pause the non-leaders + for unit in non_leaders: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + _app = subordinate.split('/')[0] + if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: + logging.info("Skipping pausing {} - blacklisted" + .format(subordinate)) + else: + logging.info("Pausing {}".format(subordinate)) + await model.async_run_action( + subordinate, "pause", action_params={}) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + await model.async_run_action(unit, "pause", action_params={}) + + machine = status["units"][leader]["machine"] + # Series upgrade the leader + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + await async_series_upgrade(leader, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + await os_utils.async_set_origin(application, origin) + await wait_for_unit_idle(unit) + + # Series upgrade the non-leaders + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + await async_series_upgrade(unit, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + await os_utils.async_set_origin(application, origin) + await wait_for_unit_idle(unit) + + +async def wait_for_unit_idle(unit_name): + app = unit_name.split('/')[0] + try: + await model.async_block_until(_unit_idle(app, unit_name), + timeout=600) + except concurrent.futures._base.TimeoutError: + raise ModelTimeout("Zaza has timed out waiting on the unit to " + "reach idle state.") + +def _unit_idle(app, unit_name): + async def f(): + x = await get_agent_status(app, unit_name) + return x == "idle" + return f + #return await get_agent_status(app, unit_name) is "idle" + +async def get_agent_status(app, unit_name): + return (await model.async_get_status()). \ + applications[app]['units'][unit_name] \ + ['agent-status']['status'] + + def series_upgrade(unit_name, machine_num, from_series="trusty", to_series="xenial", origin='openstack-origin', @@ -305,42 +505,93 @@ async def async_series_upgrade(unit_name, machine_num, application = unit_name.split('/')[0] await os_utils.async_set_dpkg_non_interactive_on_unit(unit_name) await async_dist_upgrade(unit_name) - await model.async_block_until_all_units_idle() + await wait_for_unit_idle(unit_name) logging.info("Prepare series upgrade on {}".format(machine_num)) - await model.async_prepare_series_upgrade(machine_num, to_series=to_series) + await async_prepare_series_upgrade(machine_num, to_series=to_series) logging.info("Waiting for workload status 'blocked' on {}" .format(unit_name)) await model.async_block_until_unit_wl_status(unit_name, "blocked") - logging.info("Waiting for model idleness") - await model.async_block_until_all_units_idle() + logging.info("Waiting for unit {} idleness".format(unit_name)) + await wait_for_unit_idle(unit_name) await async_wrap_do_release_upgrade(unit_name, from_series=from_series, to_series=to_series, files=files, workaround_script=workaround_script) logging.info("Reboot {}".format(unit_name)) - os_utils.reboot(unit_name) + await os_utils.async_reboot(unit_name) logging.info("Waiting for workload status 'blocked' on {}" .format(unit_name)) await model.async_block_until_unit_wl_status(unit_name, "blocked") - logging.info("Waiting for model idleness") - await model.async_block_until_all_units_idle() + logging.info("Waiting for unit {} idleness".format(unit_name)) + await wait_for_unit_idle(unit_name) logging.info("Set origin on {}".format(application)) # Allow for charms which have neither source nor openstack-origin if origin: await os_utils.async_set_origin(application, origin) - await model.async_block_until_all_units_idle() + await wait_for_unit_idle(unit_name) logging.info("Complete series upgrade on {}".format(machine_num)) - await model.async_complete_series_upgrade(machine_num) - await model.async_block_until_all_units_idle() + await async_complete_series_upgrade(machine_num) + await wait_for_unit_idle(unit_name) logging.info("Running run_post_upgrade_functions {}".format( post_upgrade_functions)) run_post_upgrade_functions(post_upgrade_functions) logging.info("Waiting for workload status 'active' on {}" .format(unit_name)) await model.async_block_until_unit_wl_status(unit_name, "active") - await model.async_block_until_all_units_idle() + await wait_for_unit_idle(unit_name) # This step may be performed by juju in the future logging.info("Set series on {} to {}".format(application, to_series)) - await model.async_set_series(application, to_series) + await async_set_series(application, to_series) + + +async def async_prepare_series_upgrade(machine_num, to_series="xenial"): + """Execute juju series-upgrade prepare on machine. + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + :param machine_num: Machine number + :type machine_num: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = await model.async_get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + machine_num, "prepare", to_series, "--yes"] + logging.info("About to call '{}'".format(cmd)) + await os_utils.check_call(cmd) + + +async def async_complete_series_upgrade(machine_num): + """Execute juju series-upgrade complete on machine. + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + :param machine_num: Machine number + :type machine_num: str + :returns: None + :rtype: None + """ + juju_model = await model.async_get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + machine_num, "complete"] + await os_utils.check_call(cmd) + + +async def async_set_series(application, to_series): + """Execute juju set-series complete on application. + NOTE: This is a new feature in juju and not yet in libjuju. + :param application: Name of application to upgrade series + :type application: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = await model.async_get_juju_model() + cmd = ["juju", "set-series", "-m", juju_model, + application, to_series] + await os_utils.check_call(cmd) def wrap_do_release_upgrade(unit_name, from_series="trusty", to_series="xenial", @@ -416,10 +667,10 @@ async def async_wrap_do_release_upgrade(unit_name, from_series="trusty", # Run Script if workaround_script: logging.info("Running workaround script") - os_utils.run_via_ssh(unit_name, workaround_script) + await os_utils.async_run_via_ssh(unit_name, workaround_script) # Actually do the do_release_upgrade - do_release_upgrade(unit_name) + await async_do_release_upgrade(unit_name) def dist_upgrade(unit_name): @@ -477,3 +728,21 @@ def do_release_upgrade(unit_name): unit_name, 'DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive') + + +async def async_do_release_upgrade(unit_name): + """Run do-release-upgrade noninteractive. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Upgrading ' + unit_name) + # NOTE: It is necessary to run this via juju ssh rather than juju run due + # to timeout restrictions and error handling. + await os_utils.async_run_via_ssh( + unit_name, + 'DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive') + diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 445d529..8851b6f 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -13,18 +13,109 @@ # limitations under the License. """Collection of functions to support upgrade testing.""" +import re +import logging +import collections +import zaza.model -SERVICE_GROUPS = { - 'Core Identity': ['keystone'], - 'Storage': [ - 'ceph-mon', 'ceph-osd', 'ceph-fs', 'ceph-radosgw', 'swift-proxy', - 'swift-storage'], - 'Control Plane': [ - 'aodh', 'barbican', 'ceilometer', 'cinder', 'designate', +SERVICE_GROUPS = collections.OrderedDict([ + ('Core Identity', ['keystone']), + ('Control Plane', [ + 'aodh', 'barbican', 'ceilometer', 'ceph-mon', 'ceph-fs', + 'ceph-radosgw', 'cinder', 'designate', 'designate-bind', 'glance', 'gnocchi', 'heat', 'manila', 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', - 'nova-cloud-controller', 'openstack-dashboard'], - 'Compute': ['nova-compute']} + 'nova-cloud-controller', 'openstack-dashboard']), + ('Data Plane', [ + 'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage']) +]) UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] + +# Series upgrade ordering should be: [ +# UPGRADE_EXCLUDE_LIST, +# SERVICE_GROUPS['Core Identity'], +# SERVICE_GROUPS['Control Plane'] + SERVICE_GROUPS['Storage'], +# SERVICE_GROUPS['Data Plane'], +# ] + +def get_upgrade_candidates(model_name=None): + """Extract list of apps from model that can be upgraded. + + :param model_name: Name of model to query. + :type model_name: str + :returns: List of application that can have their payload upgraded. + :rtype: [] + """ + status = zaza.model.get_status(model_name=model_name) + candidates = {} + for app, app_config in status.applications.items(): + # Filter out subordinates + if app_config.get("subordinate-to"): + logging.warning( + "Excluding {} from upgrade, it is a subordinate".format(app)) + continue + + # Filter out charms on the naughty list + charm_name = extract_charm_name_from_url(app_config['charm']) + if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: + logging.warning( + "Excluding {} from upgrade, on the exclude list".format(app)) + continue + + # Filter out charms that have no source option + charm_options = zaza.model.get_application_config( + app, model_name=model_name).keys() + src_options = ['openstack-origin', 'source'] + if not [x for x in src_options if x in charm_options]: + logging.warning( + "Excluding {} from upgrade, no src option".format(app)) + continue + + candidates[app] = app_config + return candidates + + +def get_upgrade_groups(model_name=None): + """Place apps in the model into their upgrade groups. + + Place apps in the model into their upgrade groups. If an app is deployed + but is not in SERVICE_GROUPS then it is placed in a sweep_up group. + + :param model_name: Name of model to query. + :type model_name: str + :returns: Dict of group lists keyed on group name. + :rtype: collections.OrderedDict + """ + apps_in_model = get_upgrade_candidates(model_name=model_name) + + groups = collections.OrderedDict() + for phase_name, charms in SERVICE_GROUPS.items(): + group = [] + for app, app_config in apps_in_model.items(): + charm_name = extract_charm_name_from_url(app_config['charm']) + if charm_name in charms: + group.append(app) + groups[phase_name] = group + + sweep_up = [] + for app in apps_in_model: + if not (app in [a for group in groups.values() for a in group]): + sweep_up.append(app) + + groups['sweep_up'] = sweep_up + return groups + +def extract_charm_name_from_url(charm_url): + """Extract the charm name from the charm url. + + E.g. Extract 'heat' from local:bionic/heat-12 + + :param charm_url: Name of model to query. + :type charm_url: str + :returns: Charm name + :rtype: str + """ + charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) + return charm_name.split(':')[-1]