From c06cb8d62513bb5962a878e3d304066143c3b959 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 19 May 2020 07:47:55 -0700 Subject: [PATCH] MySQL 8 Scale-in and Scale-out tests (#275) * MySQL 8 Scale-in and Scale-out tests Depends-On: https://github.com/openstack-charmers/zaza/pull/353 * Review requests * Typo --- zaza/openstack/charm_tests/mysql/tests.py | 166 ++++++++++++++++++++-- 1 file changed, 155 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 77bc966..5088010 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -71,6 +71,45 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest): self.non_leaders.append(unit) return self.leader, self.non_leaders + def get_cluster_status(self): + """Get cluster status. + + Return cluster status dict from the cluster-status action or raise + assertion error. + + :returns: Dictionary of cluster status + :rtype: dict + """ + logging.info("Running cluster-status action") + action = zaza.model.run_action_on_leader( + self.application, + "cluster-status", + action_params={}) + assert action.data.get("results") is not None, ( + "Cluster status action failed: No results: {}" + .format(action.data)) + assert action.data["results"].get("cluster-status") is not None, ( + "Cluster status action failed: No cluster-status: {}" + .format(action.data)) + return json.loads(action.data["results"]["cluster-status"]) + + def get_rw_primary_node(self): + """Get RW primary node. + + Return RW primary node unit. + + :returns: Unit object of primary node + :rtype: Union[Unit, None] + """ + _status = self.get_cluster_status() + _primary_ip = _status['groupInformationSourceMember'] + if ":" in _primary_ip: + _primary_ip = _primary_ip.split(':')[0] + units = zaza.model.get_units(self.application_name) + for unit in units: + if _primary_ip in unit.public_address: + return unit + class MySQLCommonTests(MySQLBaseTest): """Common mysql charm tests.""" @@ -412,14 +451,10 @@ class MySQLInnoDBClusterTests(MySQLCommonTests): Run the cluster-status action. """ logging.info("Execute cluster-status action") - action = zaza.model.run_action_on_leader( - self.application, - "cluster-status", - action_params={}) - cluster_status = json.loads(action.data["results"]["cluster-status"]) + cluster_status = self.get_cluster_status() assert "OK" in cluster_status["defaultReplicaSet"]["status"], ( - "Cluster status action failed: {}" - .format(action.data)) + "Cluster status is not OK: {}" + .format(cluster_status)) logging.info("Passed cluster-status action test.") def test_110_mysqldump(self): @@ -538,10 +573,15 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") - action = zaza.model.run_action_on_leader( - self.application, - "reboot-cluster-from-complete-outage", - action_params={}) + # We do not know which unit has the most up to date data + # run reboot-cluster-from-complete-outage until we get a success. + for unit in zaza.model.get_units(self.application): + action = zaza.model.run_action( + unit.entity_id, + "reboot-cluster-from-complete-outage", + action_params={}) + if action.data["results"].get("outcome"): + break assert "Success" in action.data["results"]["outcome"], ( "Reboot cluster from complete outage action failed: {}" .format(action.data)) @@ -670,3 +710,107 @@ class MySQL8MigrationTests(MySQLBaseTest): test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) + + +class MySQLInnoDBClusterScaleTest(MySQLBaseTest): + """Percona Cluster cold start tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running mysql-innodb-cluster scale tests.""" + super().setUpClass() + cls.application = "mysql-innodb-cluster" + cls.test_config = lifecycle_utils.get_charm_config(fatal=False) + cls.states = cls.test_config.get("target_deploy_status", {}) + + def test_800_remove_leader(self): + """Remove leader node. + + We start with a three node cluster, remove one, down to two. + The cluster will be in waiting state. + """ + logging.info("Scale in test: remove leader") + leader, nons = self.get_leaders_and_non_leaders() + leader_unit = zaza.model.get_unit_from_name(leader) + zaza.model.destroy_unit(self.application_name, leader) + + logging.info("Wait units are waiting ...") + zaza.model.block_until_unit_wl_status(nons[0], "waiting") + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info( + "Removing old unit from cluster: {} " + .format(leader_unit.public_address)) + action = zaza.model.run_action( + nons[0], + "remove-instance", + action_params={ + "address": leader_unit.public_address, + "force": True}) + assert action.data.get("results") is not None, ( + "Remove instance action failed: No results: {}" + .format(action.data)) + + def test_801_add_unit(self): + """Add mysql-innodb-cluster node. + + We start with two node cluster in waiting, add one, back to a full + cluster of three. + """ + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info("Adding unit after removed unit ...") + zaza.model.add_unit(self.application_name) + + logging.info("Wait for application states ...") + zaza.model.wait_for_application_states(states=self.states) + + def test_802_add_unit(self): + """Add another mysql-innodb-cluster node. + + We start with a three node full cluster, add another, up to a four node + cluster. + """ + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info("Adding unit after full cluster ...") + zaza.model.add_unit(self.application_name) + + logging.info("Wait for application states ...") + zaza.model.wait_for_application_states(states=self.states) + + def test_803_remove_fourth(self): + """Remove mysql-innodb-cluster node. + + We start with a four node full cluster, remove one, down to a three + node full cluster. + """ + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + leader, nons = self.get_leaders_and_non_leaders() + non_leader_unit = zaza.model.get_unit_from_name(nons[0]) + zaza.model.destroy_unit(self.application_name, nons[0]) + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info("Scale in test: back down to three") + zaza.model.wait_for_application_states(states=self.states) + + logging.info( + "Removing old unit from cluster: {} " + .format(non_leader_unit.public_address)) + action = zaza.model.run_action( + leader, + "remove-instance", + action_params={ + "address": non_leader_unit.public_address, + "force": True}) + assert action.data.get("results") is not None, ( + "Remove instance action failed: No results: {}" + .format(action.data))