From 0a94b97cf412fc5874fb01fa1794936cd2cf7dcd Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 18 Mar 2020 12:55:26 +0100 Subject: [PATCH] update for style bits --- .../test_zaza_utilities_openstack_upgrade.py | 1 - .../charm_tests/series_upgrade/tests.py | 49 +++++----- zaza/openstack/utilities/generic.py | 7 ++ zaza/openstack/utilities/openstack_upgrade.py | 3 - zaza/openstack/utilities/series_upgrade.py | 92 ++++++++++++------- zaza/openstack/utilities/upgrade_utils.py | 7 +- 6 files changed, 93 insertions(+), 66 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py index bed43b3..4d619c5 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy import mock import unit_tests.utils as ut_utils diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 1ade8d0..9d10c1a 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -49,7 +49,7 @@ class SeriesUpgradeTest(unittest.TestCase): applications = model.get_status().applications completed_machines = [] - for application in applications: + for application, app_details in applications: # Defaults origin = "openstack-origin" pause_non_leader_subordinate = True @@ -240,7 +240,6 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): if app in applications.keys()] upgrade_groups['deferred'] = [] completed_machines = [] - deferred_applications = [] for group_name, group in upgrade_groups.items(): logging.warn("About to upgrade {} ({})".format(group_name, group)) upgrade_group = [] @@ -253,11 +252,11 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): name = upgrade_utils.extract_charm_name_from_url( app_details['charm']) if name not in group and application not in group: - if group_name is not "deferred" and \ + if group_name != "deferred" and \ name not in upgrade_groups['deferred']: upgrade_groups['deferred'].append(name) continue - if group_name is not "deferred" and \ + if group_name != "deferred" and \ name in upgrade_groups['deferred']: upgrade_groups['deferred'].remove(name) # Skip subordinates @@ -305,27 +304,30 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): if "mongodb" in app_details["charm"]: # Mongodb needs to run series upgrade # on its secondaries first. - upgrade_group.append(series_upgrade_utils.async_series_upgrade_non_leaders_first( - application, - from_series=self.from_series, - to_series=self.to_series, - completed_machines=completed_machines, - post_upgrade_functions=post_upgrade_functions)) + upgrade_group.append( + series_upgrade_utils + .async_series_upgrade_non_leaders_first( + application, + from_series=self.from_series, + to_series=self.to_series, + completed_machines=completed_machines, + post_upgrade_functions=post_upgrade_functions)) continue # The rest are likley APIs use defaults - - upgrade_group.append(series_upgrade_utils.async_series_upgrade_application( - application, - pause_non_leader_primary=pause_non_leader_primary, - pause_non_leader_subordinate=pause_non_leader_subordinate, - from_series=self.from_series, - to_series=self.to_series, - origin=origin, - completed_machines=completed_machines, - workaround_script=self.workaround_script, - files=self.files, - post_upgrade_functions=post_upgrade_functions)) + pause = pause_non_leader_subordinate + upgrade_group.append( + series_upgrade_utils.async_series_upgrade_application( + application, + pause_non_leader_primary=pause_non_leader_primary, + pause_non_leader_subordinate=pause, + from_series=self.from_series, + to_series=self.to_series, + origin=origin, + completed_machines=completed_machines, + workaround_script=self.workaround_script, + files=self.files, + post_upgrade_functions=post_upgrade_functions)) asyncio.get_event_loop().run_until_complete( asyncio.gather(*upgrade_group)) @@ -348,7 +350,6 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): model.block_until_all_units_idle() - class ParallelTrustyXenialSeriesUpgrade(ParallelSeriesUpgradeTest): """Trusty to Xenial Series Upgrade. @@ -376,6 +377,6 @@ class ParallelXenialBionicSeriesUpgrade(ParallelSeriesUpgradeTest): cls.from_series = "xenial" cls.to_series = "bionic" + if __name__ == "__main__": unittest.main() - diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 596feea..a63abc5 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -328,6 +328,13 @@ async def async_reboot(unit_name): async def check_call(cmd): + """Asynchronous function to check a subprocess call. + + :param cmd: Command to execute + :type cmd: List[str] + :returns: None + :rtype: None + """ proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 87a8159..9ad4d50 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -22,9 +22,6 @@ import zaza.openstack.utilities.juju as juju_utils import zaza.model from zaza import sync_wrapper from zaza.openstack.utilities.upgrade_utils import ( - SERVICE_GROUPS, - UPGRADE_EXCLUDE_LIST, - get_upgrade_candidates, get_upgrade_groups, ) diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index c1f2cd5..e32a06f 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -14,6 +14,7 @@ """Collection of functions for testing series upgrade.""" +import concurrent import logging import os @@ -101,7 +102,8 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", model.block_until_all_units_idle() -async def async_series_upgrade_non_leaders_first(application, from_series="trusty", +async def async_series_upgrade_non_leaders_first(application, + from_series="trusty", to_series="xenial", completed_machines=[], post_upgrade_functions=None): @@ -137,11 +139,12 @@ async def async_series_upgrade_non_leaders_first(application, from_series="trust if machine not in completed_machines: logging.info("Series upgrade non-leader unit: {}" .format(unit)) - await async_series_upgrade(unit, machine, - from_series=from_series, to_series=to_series, - origin=None, - post_upgrade_functions=post_upgrade_functions) - await async_run_post_upgrade_functions(post_upgrade_functions) + await async_series_upgrade( + unit, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + run_post_upgrade_functions(post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded. " @@ -152,10 +155,11 @@ async def async_series_upgrade_non_leaders_first(application, from_series="trust machine = status["units"][leader]["machine"] logging.info("Series upgrade leader: {}".format(leader)) if machine not in completed_machines: - await async_series_upgrade(leader, machine, - from_series=from_series, to_series=to_series, - origin=None, - post_upgrade_functions=post_upgrade_functions) + await async_series_upgrade( + leader, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded." @@ -348,13 +352,14 @@ async def async_series_upgrade_application(application, # Series upgrade the leader logging.info("Series upgrade leader: {}".format(leader)) if machine not in completed_machines: - await async_series_upgrade(leader, machine, - from_series=from_series, - to_series=to_series, - origin=origin, - workaround_script=workaround_script, - files=files, - post_upgrade_functions=post_upgrade_functions) + await async_series_upgrade( + leader, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded." @@ -370,13 +375,14 @@ async def async_series_upgrade_application(application, if machine not in completed_machines: logging.info("Series upgrade non-leader unit: {}" .format(unit)) - await async_series_upgrade(unit, machine, - from_series=from_series, - to_series=to_series, - origin=origin, - workaround_script=workaround_script, - files=files, - post_upgrade_functions=post_upgrade_functions) + await async_series_upgrade( + unit, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded. " @@ -387,26 +393,44 @@ async def async_series_upgrade_application(application, await wait_for_unit_idle(unit) +# TODO: Move these functions into zaza.model async def wait_for_unit_idle(unit_name): + """Wait until the unit's agent is idle. + + :param unit_name: The unit name of the application, ex: mysql/0 + :type unit_name: str + :returns: None + :rtype: None + """ app = unit_name.split('/')[0] try: - await model.async_block_until(_unit_idle(app, unit_name), + await model.async_block_until( + _unit_idle(app, unit_name), timeout=600) except concurrent.futures._base.TimeoutError: - raise ModelTimeout("Zaza has timed out waiting on the unit to " - "reach idle state.") + raise model.ModelTimeout("Zaza has timed out waiting on the unit to " + "reach idle state.") + def _unit_idle(app, unit_name): async def f(): x = await get_agent_status(app, unit_name) return x == "idle" return f - #return await get_agent_status(app, unit_name) is "idle" + async def get_agent_status(app, unit_name): + """Get the current status of the specified unit. + + :param app: The name of the Juju application, ex: mysql + :type app: str + :param unit_name: The unit name of the application, ex: mysql/0 + :type unit_name: str + :returns: The agent status, either active / idle, returned by Juju + :rtype: str + """ return (await model.async_get_status()). \ - applications[app]['units'][unit_name] \ - ['agent-status']['status'] + applications[app]['units'][unit_name]['agent-status']['status'] def series_upgrade(unit_name, machine_num, @@ -545,6 +569,7 @@ async def async_series_upgrade(unit_name, machine_num, async def async_prepare_series_upgrade(machine_num, to_series="xenial"): """Execute juju series-upgrade prepare on machine. + NOTE: This is a new feature in juju behind a feature flag and not yet in libjuju. export JUJU_DEV_FEATURE_FLAGS=upgrade-series @@ -564,6 +589,7 @@ async def async_prepare_series_upgrade(machine_num, to_series="xenial"): async def async_complete_series_upgrade(machine_num): """Execute juju series-upgrade complete on machine. + NOTE: This is a new feature in juju behind a feature flag and not yet in libjuju. export JUJU_DEV_FEATURE_FLAGS=upgrade-series @@ -580,6 +606,7 @@ async def async_complete_series_upgrade(machine_num): async def async_set_series(application, to_series): """Execute juju set-series complete on application. + NOTE: This is a new feature in juju and not yet in libjuju. :param application: Name of application to upgrade series :type application: str @@ -593,6 +620,7 @@ async def async_set_series(application, to_series): application, to_series] await os_utils.check_call(cmd) + def wrap_do_release_upgrade(unit_name, from_series="trusty", to_series="xenial", files=None, workaround_script=None): @@ -662,7 +690,8 @@ async def async_wrap_do_release_upgrade(unit_name, from_series="trusty", logging.info("SCP files") for _file in files: logging.info("SCP {}".format(_file)) - await model.async_scp_to_unit(unit_name, _file, os.path.basename(_file)) + await model.async_scp_to_unit( + unit_name, _file, os.path.basename(_file)) # Run Script if workaround_script: @@ -746,4 +775,3 @@ async def async_do_release_upgrade(unit_name): 'DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive', raise_exceptions=True) - diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 8851b6f..4e2fbf5 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -33,12 +33,6 @@ SERVICE_GROUPS = collections.OrderedDict([ UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] -# Series upgrade ordering should be: [ -# UPGRADE_EXCLUDE_LIST, -# SERVICE_GROUPS['Core Identity'], -# SERVICE_GROUPS['Control Plane'] + SERVICE_GROUPS['Storage'], -# SERVICE_GROUPS['Data Plane'], -# ] def get_upgrade_candidates(model_name=None): """Extract list of apps from model that can be upgraded. @@ -107,6 +101,7 @@ def get_upgrade_groups(model_name=None): groups['sweep_up'] = sweep_up return groups + def extract_charm_name_from_url(charm_url): """Extract the charm name from the charm url.