From a1e5649d127fd97f33683134916420a811dec7e9 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 24 Mar 2020 16:58:37 +0100 Subject: [PATCH] Refactor if-statement into upgrade config generator --- .../charm_tests/series_upgrade/tests.py | 165 +++++------------- zaza/openstack/utilities/series_upgrade.py | 35 +--- zaza/openstack/utilities/upgrade_utils.py | 101 ++++++++++- 3 files changed, 133 insertions(+), 168 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 941e000..0470c26 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -24,14 +24,29 @@ import unittest from zaza import model from zaza.openstack.utilities import ( cli as cli_utils, - series_upgrade as series_upgrade_utils, upgrade_utils as upgrade_utils, ) from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest +def _filter_easyrsa(app, app_config, model_name=None): + logging.warn("Skipping series upgrade of easyrsa Bug #1850121") + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "easyrsa" in charm_name: + return True + return False + + +def _filter_etcd(app, app_config, model_name=None): + logging.warn("Skipping series upgrade of easyrsa Bug #1850124") + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "etcd" in charm_name: + return True + return False + + class SeriesUpgradeTest(unittest.TestCase): - """Class to encapsulate Sereis Upgrade Tests.""" + """Class to encapsulate Series Upgrade Tests.""" @classmethod def setUpClass(cls): @@ -50,75 +65,31 @@ class SeriesUpgradeTest(unittest.TestCase): applications = model.get_status().applications completed_machines = [] for application, app_details in applications: - # Defaults - origin = "openstack-origin" - pause_non_leader_subordinate = True - pause_non_leader_primary = True - post_upgrade_functions = [] # Skip subordinates if app_details["subordinate-to"]: continue if "easyrsa" in app_details["charm"]: - logging.warn("Skipping series upgrade of easyrsa Bug #1850121") + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850121") continue if "etcd" in app_details["charm"]: - logging.warn("Skipping series upgrade of easyrsa Bug #1850124") + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850124") continue - if "percona-cluster" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = True - if "rabbitmq-server" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = False - if "nova-compute" in app_details["charm"]: - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "ceph" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "designate-bind" in app_details["charm"]: - origin = None - if "tempest" in app_details["charm"]: - origin = None - if "memcached" in app_details["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "vault" in app_details["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = True - post_upgrade_functions = [ - ('zaza.openstack.charm_tests.vault.setup.' - 'mojo_unseal_by_unit')] - if "mongodb" in app_details["charm"]: - # Mongodb needs to run series upgrade - # on its secondaries first. - series_upgrade_utils.series_upgrade_non_leaders_first( - application, - from_series=self.from_series, - to_series=self.to_series, - completed_machines=completed_machines, - post_upgrade_functions=post_upgrade_functions) - continue - - # The rest are likley APIs use defaults - - series_upgrade_utils.series_upgrade_application( + upgrade_config = upgrade_utils.app_config( + app_details['charm'], + async=False) + upgrade_function = upgrade_config.pop('upgrade_function') + logging.warn("About to upgrade {}".format(application)) + upgrade_function( application, - pause_non_leader_primary=pause_non_leader_primary, - pause_non_leader_subordinate=pause_non_leader_subordinate, + **upgrade_config, from_series=self.from_series, to_series=self.to_series, - origin=origin, completed_machines=completed_machines, workaround_script=self.workaround_script, files=self.files, - post_upgrade_functions=post_upgrade_functions) - + ) if "rabbitmq-server" in app_details["charm"]: logging.info( "Running complete-cluster-series-upgrade action on leader") @@ -218,7 +189,7 @@ class XenialBionicSeriesUpgrade(SeriesUpgradeTest): class ParallelSeriesUpgradeTest(unittest.TestCase): - """Class to encapsulate Sereis Upgrade Tests.""" + """Class to encapsulate Series Upgrade Tests.""" @classmethod def setUpClass(cls): @@ -233,88 +204,30 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): """Run series upgrade.""" # Set Feature Flag os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" - upgrade_groups = upgrade_utils.get_series_upgrade_groups() + upgrade_groups = upgrade_utils.get_series_upgrade_groups( + extra_filters=[_filter_etcd, _filter_easyrsa]) applications = model.get_status().applications completed_machines = [] for group_name, group in upgrade_groups.items(): logging.warn("About to upgrade {} ({})".format(group_name, group)) upgrade_group = [] for application, app_details in applications.items(): - # Defaults - origin = "openstack-origin" - pause_non_leader_subordinate = True - pause_non_leader_primary = True - post_upgrade_functions = [] - # Skip subordinates - if app_details["subordinate-to"]: - continue - if "easyrsa" in app_details["charm"]: - logging.warn( - "Skipping series upgrade of easyrsa Bug #1850121") - continue - if "etcd" in app_details["charm"]: - logging.warn( - "Skipping series upgrade of easyrsa Bug #1850124") + if application not in group: continue + upgrade_config = upgrade_utils.app_config(app_details['charm']) + upgrade_function = upgrade_config.pop('upgrade_function') logging.warn("About to upgrade {}".format(application)) - if "percona-cluster" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = True - if "rabbitmq-server" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = False - if "nova-compute" in app_details["charm"]: - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "ceph" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "designate-bind" in app_details["charm"]: - origin = None - if "tempest" in app_details["charm"]: - origin = None - if "memcached" in app_details["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "vault" in app_details["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = True - post_upgrade_functions = [ - ('zaza.openstack.charm_tests.vault.setup.' - 'mojo_unseal_by_unit')] - if "mongodb" in app_details["charm"]: - # Mongodb needs to run series upgrade - # on its secondaries first. - upgrade_group.append( - series_upgrade_utils - .async_series_upgrade_non_leaders_first( - application, - from_series=self.from_series, - to_series=self.to_series, - completed_machines=completed_machines, - post_upgrade_functions=post_upgrade_functions)) - continue - - # The rest are likley APIs use defaults - pause = pause_non_leader_subordinate + logging.info("\tConfig: {}".format(upgrade_config)) upgrade_group.append( - series_upgrade_utils.async_series_upgrade_application( + upgrade_function( application, - pause_non_leader_primary=pause_non_leader_primary, - pause_non_leader_subordinate=pause, + **upgrade_config, from_series=self.from_series, to_series=self.to_series, - origin=origin, completed_machines=completed_machines, workaround_script=self.workaround_script, files=self.files, - post_upgrade_functions=post_upgrade_functions)) - + )) asyncio.get_event_loop().run_until_complete( asyncio.gather(*upgrade_group)) if "rabbitmq-server" in group: diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index f217b17..6b5fd2b 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -42,40 +42,6 @@ def run_post_upgrade_functions(post_upgrade_functions): cl_utils.get_class(func)() -def get_charm_settings(): - """Application series upgrade settings. - - :returns: A Dict of the settings to manage a series upgrade - :rtype: Dict - - """ - default = { - 'origin': 'openstack-origin', - 'pause_non_leader_subordinate': True, - 'pause_non_leader_primary': True, - 'upgrade_function': async_series_upgrade_application, - 'post_upgrade_functions': []} - - _app_settings = collections.defaultdict(lambda: default) - exceptions = { - 'rabbitmq-server': { - 'origin': 'source', - 'pause_non_leader_subordinate': False}, - 'percona-cluster': {'origin': 'source'}, - 'nova-compute': { - 'pause_non_leader_primary': False, - 'pause_non_leader_subordinate': False}, - 'mongodb': { - 'upgrade_function': async_series_upgrade_non_leaders_first, - 'origin': None, - - }} - for key, value in exceptions.items(): - _app_settings[key] = copy.deepcopy(default) - _app_settings[key].update(value) - return _app_settings - - def series_upgrade_non_leaders_first(application, from_series="trusty", to_series="xenial", completed_machines=[], @@ -370,6 +336,7 @@ async def async_series_upgrade_application( # generalized. leader = None non_leaders = [] + logging.info("Configuring leader / non leaders for {}".format(application)) for unit in status["units"]: if status["units"][unit].get("leader"): leader = unit diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index c611398..b1a7c50 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -90,7 +90,7 @@ def _filter_non_openstack_services(app, app_config, model_name=None): return False -def get_upgrade_groups(model_name=None): +def get_upgrade_groups(model_name=None, extra_filters=None): """Place apps in the model into their upgrade groups. Place apps in the model into their upgrade groups. If an app is deployed @@ -101,18 +101,28 @@ def get_upgrade_groups(model_name=None): :returns: Dict of group lists keyed on group name. :rtype: collections.OrderedDict """ + filters = [ + _filter_subordinates, + _filter_openstack_upgrade_list, + _filter_non_openstack_services, + ] + if extra_filters: + if isinstance(extra_filters, list): + filters.extend(extra_filters) + elif callable(extra_filters): + filters.append(extra_filters) + else: + raise RuntimeError( + "extra_filters should be a list of " + "callables") apps_in_model = get_upgrade_candidates( model_name=model_name, - filters=[ - _filter_subordinates, - _filter_openstack_upgrade_list, - _filter_non_openstack_services, - ]) + filters=filters,) return _build_service_groups(apps_in_model) -def get_series_upgrade_groups(model_name=None): +def get_series_upgrade_groups(model_name=None, extra_filters=None): """Place apps in the model into their upgrade groups. Place apps in the model into their upgrade groups. If an app is deployed @@ -123,9 +133,19 @@ def get_series_upgrade_groups(model_name=None): :returns: Dict of group lists keyed on group name. :rtype: collections.OrderedDict """ + filters = [_filter_subordinates] + if extra_filters: + if isinstance(extra_filters, list): + filters.extend(extra_filters) + elif callable(extra_filters): + filters.append(extra_filters) + else: + raise RuntimeError( + "extra_filters should be a list of " + "callables") apps_in_model = get_upgrade_candidates( model_name=model_name, - filters=[_filter_subordinates]) + filters=filters) return _build_service_groups(apps_in_model) @@ -162,3 +182,68 @@ def extract_charm_name_from_url(charm_url): """ charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) return charm_name.split(':')[-1] + + +def app_config(charm_name, async=True): + """Returns a dict with the upgrade config for an application. + + :param charm_name: Name of the charm about to upgrade + :type charm_name: str + :param async: Whether the upgreade functions should be async + :type async: bool + :returns: A dicitonary of the upgrade config for the application + :rtype: Dict + """ + if async: + default_upgrade = async_series_upgrade_application + secondary_first_upgrade = async_series_upgrade_non_leaders_first + else: + default_upgrade = series_upgrade_application + secondary_first_upgrade = series_upgrade_non_leaders_first + default = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'upgrade_function': default_upgrade, + 'post_upgrade_functions': []} + _app_settings = collections.defaultdict(lambda: default) + ceph = { + 'origin': "source", + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + } + exceptions = { + 'rabbitmq-server': { + 'origin': 'source', + 'pause_non_leader_subordinate': False, }, + 'percona-cluster': {'origin': 'source', }, + 'nova-compute' : { + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, }, + 'ceph': ceph, + 'ceph-mon': ceph, + 'ceph-osd': ceph, + 'designate-bind': {'origin': None, }, + 'tempest': {'origin': None, }, + 'memcached': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + }, + 'vault': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': True, + 'post_upgrade_functions': [ + ('zaza.openstack.charm_tests.vault.setup.' + 'mojo_unseal_by_unit')] + }, + 'mongodb': { + 'upgrade_function': secondary_first_upgrade, + } + + } + for key, value in exceptions.items(): + _app_settings[key] = copy.deepcopy(default) + _app_settings[key].update(value) + return _app_settings[charm_name]