From af2c6243e2c2ea5429f89cb4f36f3f2e216261de Mon Sep 17 00:00:00 2001 From: David Ames Date: Sat, 6 Feb 2021 00:31:28 +0000 Subject: [PATCH 1/3] Fix race condition in MySQL tests In the MySQL test we destroy a unit and then attempt to remove it from metadata. The metadata removal (remove_instance action) gets stuck when it is started while the destroyed unit has not been entirely removed and is still running MySQL. Wait for all the units to have cluster incomplete workload status message before attempting the remove_instance. --- zaza/openstack/charm_tests/mysql/tests.py | 32 +++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index d4c89ec..982f0d2 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -753,13 +753,19 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Scale in test: remove leader") leader, nons = self.get_leaders_and_non_leaders() leader_unit = zaza.model.get_unit_from_name(leader) - zaza.model.destroy_unit(self.application_name, leader) - - logging.info("Wait until unit is in waiting state ...") - zaza.model.block_until_unit_wl_status(nons[0], "waiting") + # Wait until we are idle in the hopes clients are not running + # update-status hooks logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() + zaza.model.destroy_unit(self.application_name, leader) + + logging.info("Wait until all only 2 units ...") + zaza.model.block_until_unit_count(self.application, 2) + + logging.info("Wait until all units are cluster incomplete ...") + zaza.model.block_until_wl_status_info_starts_with( + self.application, "'cluster' incomplete") logging.info( "Removing old unit from cluster: {} " @@ -786,6 +792,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Adding unit after removed unit ...") zaza.model.add_unit(self.application_name) + logging.info("Wait until 3 units ...") + zaza.model.block_until_unit_count(self.application, 3) + logging.info("Wait for application states ...") zaza.model.wait_for_application_states(states=self.states) @@ -801,6 +810,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Adding unit after full cluster ...") zaza.model.add_unit(self.application_name) + logging.info("Wait until 4 units ...") + zaza.model.block_until_unit_count(self.application, 4) + logging.info("Wait for application states ...") zaza.model.wait_for_application_states(states=self.states) @@ -810,17 +822,21 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): We start with a four node full cluster, remove one, down to a three node full cluster. """ - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() - leader, nons = self.get_leaders_and_non_leaders() non_leader_unit = zaza.model.get_unit_from_name(nons[0]) - zaza.model.destroy_unit(self.application_name, nons[0]) + # Wait until we are idle in the hopes clients are not running + # update-status hooks logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() + zaza.model.destroy_unit(self.application_name, nons[0]) + logging.info("Scale in test: back down to three") + logging.info("Wait until 3 units ...") + zaza.model.block_until_unit_count(self.application, 3) + + logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.states) logging.info( From 135da1e1a361e01cadfe17df9a25cfa92f4f1b63 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 18 Feb 2021 09:37:49 -0800 Subject: [PATCH 2/3] Fix reboot-cluster-from-complete-outage race --- zaza/openstack/charm_tests/mysql/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 982f0d2..6322684 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -589,6 +589,10 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): unit.entity_id, 'blocked') + # Wait until update-status hooks have completed + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") # We do not know which unit has the most up to date data From adda37269958edad8c6087d264e58b8267f5261e Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 2 Mar 2021 22:44:40 +0000 Subject: [PATCH 3/3] Show cluster status before removal --- zaza/openstack/charm_tests/mysql/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 6322684..15256e7 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -771,6 +771,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): zaza.model.block_until_wl_status_info_starts_with( self.application, "'cluster' incomplete") + # Show status + logging.info(self.get_cluster_status()) + logging.info( "Removing old unit from cluster: {} " .format(leader_unit.public_address)) @@ -843,6 +846,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.states) + # Show status + logging.info(self.get_cluster_status()) + logging.info( "Removing old unit from cluster: {} " .format(non_leader_unit.public_address))