From 06f181beaacccdc520c3b30ee5242b9107e12d61 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 5 Oct 2018 11:03:25 -0700 Subject: [PATCH 1/3] Juju daemons remain up after pre-series-upgrade The latest edge version of juju has the juju daemons remain up after the completion of the pre-series-upgrade hook. This is a slight change in behavior which requires a change to the zaza process. But it also allows us to test for model idleness which we could not do when the agents were down. This should avoid more race conditions. --- zaza/utilities/generic.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index c062e2f..5322f57 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -237,8 +237,12 @@ def series_upgrade_application(application, pause_non_leader_primary=True, files=files) completed_machines.append(machine) else: - logging.info("Skipping unit: {}. Machine: {} already upgraded" - .format(unit, machine)) + logging.info("Skipping unit: {}. Machine: {} already upgraded." + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + set_origin(application, origin) + model.block_until_all_units_idle() # Series upgrade the non-leaders for unit in non_leaders: @@ -252,8 +256,12 @@ def series_upgrade_application(application, pause_non_leader_primary=True, files=files) completed_machines.append(machine) else: - logging.info("Skipping unit: {}. Machine: {} already upgraded" - .format(unit, machine)) + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + set_origin(application, origin) + model.block_until_all_units_idle() def series_upgrade(unit_name, machine_num, @@ -285,9 +293,11 @@ def series_upgrade(unit_name, machine_num, set_dpkg_non_interactive_on_unit(unit_name) logging.info("Prepare series upgrade on {}".format(machine_num)) model.prepare_series_upgrade(machine_num, to_series=to_series) - logging.info("Watiing for workload status 'unknown' on {}" + logging.info("Watiing for workload status 'blocked' on {}" .format(unit_name)) - model.block_until_unit_wl_status(unit_name, "unknown") + model.block_until_unit_wl_status(unit_name, "blocked") + logging.info("Watiing for model idleness") + model.block_until_all_units_idle() wrap_do_release_upgrade(unit_name, from_series=from_series, to_series=to_series, files=files, workaround_script=workaround_script) From 202ab0b86ba9094352211d400c68d51c336b48df Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 5 Oct 2018 14:50:10 -0700 Subject: [PATCH 2/3] Fix completed machines --- zaza/charm_tests/series_upgrade/tests.py | 2 ++ zaza/utilities/generic.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/zaza/charm_tests/series_upgrade/tests.py b/zaza/charm_tests/series_upgrade/tests.py index bf1aa6a..2c766cf 100644 --- a/zaza/charm_tests/series_upgrade/tests.py +++ b/zaza/charm_tests/series_upgrade/tests.py @@ -46,6 +46,7 @@ class SeriesUpgradeTest(unittest.TestCase): os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" applications = model.get_status().applications + completed_machines = [] for application in applications: # Defaults origin = "openstack-origin" @@ -78,6 +79,7 @@ class SeriesUpgradeTest(unittest.TestCase): from_series=self.from_series, to_series=self.to_series, origin=origin, + completed_machines=completed_machines, workaround_script=self.workaround_script, files=self.files) diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index 5322f57..2d53c39 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -172,6 +172,7 @@ def series_upgrade_application(application, pause_non_leader_primary=True, pause_non_leader_subordinate=True, from_series="trusty", to_series="xenial", origin='openstack-origin', + completed_machines=[], files=None, workaround_script=None): """Series upgrade application. @@ -194,6 +195,9 @@ def series_upgrade_application(application, pause_non_leader_primary=True, :param origin: The configuration setting variable name for changing origin source. (openstack-origin or source) :type origin: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type files: list :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -202,7 +206,6 @@ def series_upgrade_application(application, pause_non_leader_primary=True, :rtype: None """ status = model.get_status().applications[application] - completed_machines = [] # For some applications (percona-cluster) the leader unit must upgrade # first. For API applications the non-leader haclusters must be paused From 58af3438ddd47c002ea1835c24fb2b737101194c Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 9 Oct 2018 09:09:25 -0700 Subject: [PATCH 3/3] Fix completed machines unit tests --- unit_tests/utilities/test_zaza_utilities_generic.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 4f06c39..46313b1 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -259,6 +259,7 @@ class TestGenericUtils(ut_utils.BaseTestCase): _origin = "source" _files = ["filename", "scriptname"] _workaround_script = "scriptname" + _completed_machines = [] # Peers and Subordinates _run_action_calls = [ mock.call("{}-hacluster/1".format(_application), @@ -283,6 +284,7 @@ class TestGenericUtils(ut_utils.BaseTestCase): to_series=_to_series, from_series=_from_series, pause_non_leader_primary=True, pause_non_leader_subordinate=True, + completed_machines=_completed_machines, workaround_script=_workaround_script, files=_files), self.run_action.assert_has_calls(_run_action_calls) self.series_upgrade.assert_has_calls(_series_upgrade_calls) @@ -296,6 +298,7 @@ class TestGenericUtils(ut_utils.BaseTestCase): _origin = "source" _files = ["filename", "scriptname"] _workaround_script = "scriptname" + _completed_machines = [] # Subordinates only _run_action_calls = [ mock.call("{}-hacluster/1".format(_application), @@ -319,6 +322,7 @@ class TestGenericUtils(ut_utils.BaseTestCase): to_series=_to_series, from_series=_from_series, pause_non_leader_primary=False, pause_non_leader_subordinate=True, + completed_machines=_completed_machines, workaround_script=_workaround_script, files=_files), self.run_action.assert_has_calls(_run_action_calls) self.series_upgrade.assert_has_calls(_series_upgrade_calls) @@ -333,6 +337,7 @@ class TestGenericUtils(ut_utils.BaseTestCase): _series_upgrade_calls = [] _files = ["filename", "scriptname"] _workaround_script = "scriptname" + _completed_machines = [] for machine_num in ("0", "1", "2"): _series_upgrade_calls.append( @@ -348,6 +353,7 @@ class TestGenericUtils(ut_utils.BaseTestCase): to_series=_to_series, from_series=_from_series, pause_non_leader_primary=False, pause_non_leader_subordinate=False, + completed_machines=_completed_machines, workaround_script=_workaround_script, files=_files) self.run_action.assert_not_called() self.series_upgrade.assert_has_calls(_series_upgrade_calls)