From 186708e88353bbb99819c6ce695a4315791337cf Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 9 Apr 2020 09:12:14 +0200 Subject: [PATCH] Change upgrade process to handle each machine fully Updating each machine in a fully self-contained way makes it easy to add per-machine and (possibly) per-application post-upgrade function calls. This still performs the actual series upgrades fully in parallel but will, for example, start the do-release-upgrade on one unit while another is still performing the initial dist-upgrade --- .../series_upgrade/parallel_tests.py | 2 +- .../utilities/parallel_series_upgrade.py | 113 ++++++++++++------ 2 files changed, 80 insertions(+), 35 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index 1b64994..7c9dba0 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -28,7 +28,7 @@ from zaza.openstack.utilities import ( upgrade_utils as upgrade_utils, ) from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest -from zaza.openstack.utilities.parallel_series_upgrade import ( +from zaza.openstack.utilities import ( parallel_series_upgrade, ) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index a959838..1585ef7 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -125,6 +125,16 @@ async def parallel_series_upgrade( :param origin: The configuration setting variable name for changing origin source. (openstack-origin or source) :type origin: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param post_upgrade_functions: A list of Zaza functions to call when + the upgrade is complete on each machine + :type post_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -170,14 +180,14 @@ async def parallel_series_upgrade( await prepare_series_upgrade(leader_machine, to_series=to_series) if leader_machine not in completed_machines: machines.append(leader_machine) - # do the dist upgrade - await dist_upgrades(machines) - # do a do-release-upgrade - await do_release_upgrades(machines) - # do a reboot - await reboots(machines) - - await complete_series_upgrade(machines, to_series) + upgrade_group = [ + series_upgrade_machine( + machine, from_series=from_series, to_series=to_series, + files=files, workaround_script=None, + post_upgrade_functions=post_upgrade_functions) + for machine in machines + ] + await asyncio.gather(*upgrade_group) if origin: await os_utils.async_set_origin(application, origin) @@ -201,6 +211,16 @@ async def serial_series_upgrade( :param origin: The configuration setting variable name for changing origin source. (openstack-origin or source) :type origin: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param post_upgrade_functions: A list of Zaza functions to call when + the upgrade is complete on each machine + :type post_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -241,42 +261,67 @@ async def serial_series_upgrade( if not follower_first and leader_machine not in completed_machines: await prepare_series_upgrade(leader_machine, to_series) - logging.info( - "About to dist-upgrade the leader ({})".format(leader_machine)) - # upgrade the leader - await dist_upgrades([leader_machine]) - # do a do-release-upgrade - await do_release_upgrades([leader_machine]) - # do a reboot - await reboots([leader_machine]) - await complete_series_upgrade([leader_machine], to_series) + logging.info("About to upgrade leader of {}: {}" + .format(application, leader_machine)) + await series_upgrade_machine( + leader_machine, from_series=from_series, to_series=to_series, + files=files, workaround_script=None, + post_upgrade_functions=post_upgrade_functions) for machine in machines: await prepare_series_upgrade(machine, to_series) - logging.debug( - "About to dist-upgrade follower {}".format(machine)) - await dist_upgrades([machine]) - # do a do-release-upgrade - await do_release_upgrades([machine]) - # do a reboot - await reboots([machine]) - await complete_series_upgrade([machine], to_series) + logging.info("About to upgrade follower of {}: {}" + .format(application, machine)) + await series_upgrade_machine( + machine, from_series=from_series, to_series=to_series, + files=files, workaround_script=None, + post_upgrade_functions=post_upgrade_functions) if follower_first and leader_machine not in completed_machines: await prepare_series_upgrade(leader_machine, to_series) - logging.info( - "About to dist-upgrade the leader ({})".format(leader_machine)) - # upgrade the leader - await dist_upgrades([leader_machine]) - # do a do-release-upgrade - await do_release_upgrades([leader_machine]) - # do a reboot - await reboots([leader_machine]) - await complete_series_upgrade([leader_machine], to_series) + logging.info("About to upgrade leader of {}: {}" + .format(application, leader_machine)) + await series_upgrade_machine( + leader_machine, from_series=from_series, to_series=to_series, + files=files, workaround_script=None, + post_upgrade_functions=post_upgrade_functions) if origin: await os_utils.async_set_origin(application, origin) +async def series_upgrade_machine( + machine, from_series='xenial', to_series='bionic', + files=None, workaround_script=None, post_upgrade_functions=None): + """Perform series upgrade on an machine. + + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :param post_upgrade_functions: A list of Zaza functions to call when + the upgrade is complete on each machine + :type post_upgrade_functions: List[str] + :returns: None + :rtype: None + """ + logging.info( + "About to dist-upgrade ({})".format(machine)) + # upgrade the do the dist upgrade + await async_dist_upgrade(machine) + # do a do-release-upgrade + await async_do_release_upgrade(machine) + # do a reboot + await reboot(machine) + await series_upgrade_utils.async_complete_series_upgrade(machine) + series_upgrade_utils.run_post_upgrade_functions(post_upgrade_functions) + + async def maybe_pause_things( status, units, pause_non_leader_subordinate=True, pause_non_leader_primary=True):