From fe8b78b7c6640ec40b1566aa8fd2d512c2da7476 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 14:08:54 -0700 Subject: [PATCH 01/10] Functional testing for mysql-innodb-cluster This is a change with functional test for mysql-innodb-cluster implementing `force-quorum-using-partition-of` feature. It is addressing bug: https://bugs.launchpad.net/charm-mysql-innodb-cluster/+bug/1917332 --- zaza/openstack/charm_tests/mysql/tests.py | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 45dbf71..7af9c75 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -617,6 +617,77 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) + def test_110_force_quorum_using_partition_of(self): + """Force quorum using partition of given address. + + After outage, cluster can end up without quorum. Force it. + """ + _machines = sorted( + juju_utils.get_machine_uuids_for_application(self.application)) + # Wait until update-status hooks have completed + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # Block all traffic across mysql instances: 0<-1, 1<-2 and 2<-0 + mysql_units = [unit for unit in zaza.model.get_units(self.application)] + no_of_units = len(mysql_units) + for index, unit in enumerate(mysql_units): + next_unit = mysql_units[(index+1)%no_of_units] + ip_address = next_unit.public_address + cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) + zaza.model.async_run_on_unit(unit, cmd) + + logging.info( + "Wait till all {} units are in state 'blocked' ..." + .format(self.application)) + for unit in zaza.model.get_units(self.application): + zaza.model.block_until_unit_wl_status( + unit.entity_id, + 'blocked', + negate_match=True) + + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + # Unblock all traffic across mysql instances + for unit in zaza.model.get_units(self.application): + cmd = "sudo iptables -F" + zaza.model.async_run_on_unit(unit, cmd) + + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + logging.info("Execute force-quorum-using-partition-of action ...") + + # Select "quorum leader" unit + leader_unit, other_units = mysql_units[0], mysq_units[1:] + action = zaza.model.run_action( + leader_unit.entity_id, + "force-quorum-using-partition-of", + action_params={"address": leader_unit.public_ip}) + + assert "Success" in action.data["results"]["outcome"], ( + "Force quorum using partition of action failed: {}" + .format(action.data)) + + # Rejoin other units to cluster + for unit in other_units: + zaza.model.run_action( + leader_unit.entity_id, + "rejoin-instance", + action_params={"address": unit.public_ip}) + + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" From a891ff28d5131c0e6bc02f38b3336bfe4eb61a0d Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 16:38:21 -0700 Subject: [PATCH 02/10] Added 'i-really-mean-it' tick to force-quorum --- zaza/openstack/charm_tests/mysql/tests.py | 29 +++++++---------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 7af9c75..22a6b5c 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -618,7 +618,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): states=test_config.get("target_deploy_status", {})) def test_110_force_quorum_using_partition_of(self): - """Force quorum using partition of given address. + """Force quorum using partition of instance with given address. After outage, cluster can end up without quorum. Force it. """ @@ -653,18 +653,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): self.resolve_update_status_errors() zaza.model.block_until_all_units_idle() - # Unblock all traffic across mysql instances - for unit in zaza.model.get_units(self.application): - cmd = "sudo iptables -F" - zaza.model.async_run_on_unit(unit, cmd) - - logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() - logging.info("Execute force-quorum-using-partition-of action ...") # Select "quorum leader" unit @@ -672,18 +660,19 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): action = zaza.model.run_action( leader_unit.entity_id, "force-quorum-using-partition-of", - action_params={"address": leader_unit.public_ip}) + action_params={ + "address": leader_unit.public_ip, + 'i-really-mean-it': True + }) assert "Success" in action.data["results"]["outcome"], ( "Force quorum using partition of action failed: {}" .format(action.data)) - # Rejoin other units to cluster - for unit in other_units: - zaza.model.run_action( - leader_unit.entity_id, - "rejoin-instance", - action_params={"address": unit.public_ip}) + # Unblock all traffic across mysql instances + for unit in zaza.model.get_units(self.application): + cmd = "sudo iptables -F" + zaza.model.async_run_on_unit(unit, cmd) for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") From e8157f0df8fab193de6ac70e2ddcd5e21145206f Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 20:44:23 -0700 Subject: [PATCH 03/10] Addressed flake8 violations --- zaza/openstack/charm_tests/mysql/tests.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 22a6b5c..4e9da67 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -622,9 +622,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): After outage, cluster can end up without quorum. Force it. """ - _machines = sorted( - juju_utils.get_machine_uuids_for_application(self.application)) - # Wait until update-status hooks have completed logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() @@ -632,7 +629,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): mysql_units = [unit for unit in zaza.model.get_units(self.application)] no_of_units = len(mysql_units) for index, unit in enumerate(mysql_units): - next_unit = mysql_units[(index+1)%no_of_units] + next_unit = mysql_units[(index+1) % no_of_units] ip_address = next_unit.public_address cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) zaza.model.async_run_on_unit(unit, cmd) @@ -656,7 +653,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): logging.info("Execute force-quorum-using-partition-of action ...") # Select "quorum leader" unit - leader_unit, other_units = mysql_units[0], mysq_units[1:] + leader_unit = mysql_units[0] action = zaza.model.run_action( leader_unit.entity_id, "force-quorum-using-partition-of", From 66fd299f16eaf5f57f6db855370faa146b59daa2 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 21:30:13 -0700 Subject: [PATCH 04/10] Fixed typo --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 4e9da67..6923bc0 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -658,7 +658,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): leader_unit.entity_id, "force-quorum-using-partition-of", action_params={ - "address": leader_unit.public_ip, + "address": leader_unit.public_address, 'i-really-mean-it': True }) From 51a53d4e864ed558f4ffafc033b683d8ca5f56a7 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 07:12:06 -0700 Subject: [PATCH 05/10] Verify at the end that cluster is in operable state --- zaza/openstack/charm_tests/mysql/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 6923bc0..6e0de25 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -671,8 +671,12 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): cmd = "sudo iptables -F" zaza.model.async_run_on_unit(unit, cmd) + logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {})) class MySQL8MigrationTests(MySQLBaseTest): From 8b47c8a37f4e9a94f0db1d6755988ccdb507f8ea Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 07:40:00 -0700 Subject: [PATCH 06/10] Change the way how I'm checking state --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 6e0de25..2226893 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -662,7 +662,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): 'i-really-mean-it': True }) - assert "Success" in action.data["results"]["outcome"], ( + assert action.data.get("results")["outcome"] is not Noe, ( "Force quorum using partition of action failed: {}" .format(action.data)) From 795272269f3f1548a9692404ea7ac5760276dd97 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 07:52:18 -0700 Subject: [PATCH 07/10] Fixed typo --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 2226893..311158a 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -662,7 +662,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): 'i-really-mean-it': True }) - assert action.data.get("results")["outcome"] is not Noe, ( + assert action.data.get("results") is not None, ( "Force quorum using partition of action failed: {}" .format(action.data)) From 9a5e334b444f1ab626aa8225f1884f6185bf7822 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 11:05:52 -0700 Subject: [PATCH 08/10] Addressed review comments --- zaza/openstack/charm_tests/mysql/tests.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 311158a..546fef3 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -644,11 +644,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): negate_match=True) logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() + zaza.model.block_until_all_units_idle() logging.info("Execute force-quorum-using-partition-of action ...") @@ -665,6 +661,9 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): assert action.data.get("results") is not None, ( "Force quorum using partition of action failed: {}" .format(action.data)) + logging.debug( + "Results from running 'force-quorum' command ...\n{}".format( + action.data) # Unblock all traffic across mysql instances for unit in zaza.model.get_units(self.application): @@ -678,6 +677,13 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" From 0858f016cdf42d9a076e9f47fe9e8d720801c261 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 11:29:44 -0700 Subject: [PATCH 09/10] Fixed TokenError --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 546fef3..219d25b 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -663,7 +663,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): .format(action.data)) logging.debug( "Results from running 'force-quorum' command ...\n{}".format( - action.data) + action.data)) # Unblock all traffic across mysql instances for unit in zaza.model.get_units(self.application): From 4df0cbee8ce6a70926d1fa953c0859409000437c Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 11:40:07 -0700 Subject: [PATCH 10/10] Moved block_until_idle immediately after force-quorum cmd --- zaza/openstack/charm_tests/mysql/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 219d25b..5cc563d 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -665,6 +665,13 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): "Results from running 'force-quorum' command ...\n{}".format( action.data)) + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + # Unblock all traffic across mysql instances for unit in zaza.model.get_units(self.application): cmd = "sudo iptables -F" @@ -677,13 +684,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) - logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() - class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests."""