From 138bd1120720167193d90e17ea1a9b23a6b6b777 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 9 Jul 2019 10:40:41 -0700 Subject: [PATCH 1/3] Zaza tests for percona-cluster Port over amulet tests to zaza for percona-cluster. Add testing of new feature actions for cold start tests: bootstrap-pxc and notify-bootstrapped. --- requirements.txt | 3 +- zaza/openstack/charm_tests/mysql/__init__.py | 14 + zaza/openstack/charm_tests/mysql/tests.py | 382 +++++++++++++++++++ zaza/openstack/charm_tests/test_utils.py | 37 +- 4 files changed, 421 insertions(+), 15 deletions(-) create mode 100644 zaza/openstack/charm_tests/mysql/__init__.py create mode 100644 zaza/openstack/charm_tests/mysql/tests.py diff --git a/requirements.txt b/requirements.txt index 307fa94..1ee936a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ PyYAML<=4.2,>=3.0 flake8>=2.2.4,<=3.5.0 flake8-docstrings flake8-per-file-ignores +pydocstyle<4.0.0 coverage mock>=1.2 nose>=1.3.7 @@ -36,4 +37,4 @@ paramiko # Documentation requirements sphinx sphinxcontrib-asyncio -git+https://github.com/openstack-charmers/zaza#egg=zaza \ No newline at end of file +git+https://github.com/openstack-charmers/zaza#egg=zaza diff --git a/zaza/openstack/charm_tests/mysql/__init__.py b/zaza/openstack/charm_tests/mysql/__init__.py new file mode 100644 index 0000000..8c7e117 --- /dev/null +++ b/zaza/openstack/charm_tests/mysql/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Collection of code for setting up and testing mysql or percona-cluster.""" diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py new file mode 100644 index 0000000..b5d4592 --- /dev/null +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -0,0 +1,382 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MySQL/Percona Cluster Testing.""" + +import logging +import os +import re +import socket +import telnetlib +import time + +import zaza.charm_lifecycle.utils as lifecycle_utils +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.juju as juju_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +class MySQLTest(test_utils.OpenStackBaseTest): + """Base for mysql charm tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running mysql tests.""" + super(MySQLTest, cls).setUpClass() + cls.application = "mysql" + cls.services = ["mysqld"] + + +class PerconaClusterTest(test_utils.OpenStackBaseTest): + """Base for percona-cluster charm tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running percona-cluster tests.""" + super(PerconaClusterTest, cls).setUpClass() + cls.application = "percona-cluster" + # This is the service pidof will attempt to find + # rather than what systemctl uses + cls.services = ["mysqld"] + cls.vip = os.environ.get("OS_VIP00") + cls.leader = None + cls.non_leaders = [] + + def get_root_password(self): + """Get the MySQL root password. + + :returns: str password + """ + return zaza.model.run_on_leader( + self.application, + "leader-get root-password")["Stdout"].strip() + + def get_wsrep_value(self, attr): + """Get wsrrep value from the DB. + + :param attr: Attribute to query + :type attr: str + :returns: str wsrep value + """ + root_password = self.get_root_password() + cmd = ("mysql -uroot -p{} -e\"show status like '{}';\"| " + "grep {}".format(root_password, attr, attr)) + output = zaza.model.run_on_leader( + self.application, cmd)["Stdout"].strip() + value = re.search(r"^.+?\s+(.+)", output).group(1) + logging.debug("%s = %s" % (attr, value)) + return value + + def is_pxc_bootstrapped(self): + """Determine if the cluster is bootstrapped. + + Query the wsrep_ready status in the DB. + + :returns: boolean + """ + value = self.get_wsrep_value("wsrep_ready") + return value.lower() in ["on", "ready"] + + def get_cluster_size(self): + """Determine the cluster size. + + Query the wsrep_cluster size in the DB. + + :returns: str Numeric cluster size + """ + return self.get_wsrep_value("wsrep_cluster_size") + + def get_crm_master(self): + """Determine CRM master for the VIP. + + Query CRM to determine which node hosts the VIP. + + :returns: str Unit name + """ + for unit in zaza.model.get_units(self.application): + # is the vip running here? + cmd = 'sudo ip a | grep "inet {}/"'.format(self.vip) + result = zaza.model.run_on_unit(unit.entity_id, cmd) + code = result.get("Code") + output = result.get("Stdout").strip() + logging.info("Checking {}".format(unit.entity_id)) + logging.debug(output) + if code == "0": + logging.info("vip ({}) running in {}".format( + self.vip, + unit.entity_id) + ) + return unit.entity_id + + def is_port_open(self, port="3306", address=None): + """Determine if MySQL is accessible. + + Connect to the MySQL port on the VIP. + + :param port: Port number + :type port: str + :param address: IP address + :type port: str + :returns: boolean + """ + try: + telnetlib.Telnet(address, port) + return True + except socket.error as e: + if e.errno == 113: + self.log.error("could not connect to %s:%s" % (address, port)) + if e.errno == 111: + self.log.error("connection refused connecting" + " to %s:%s" % (address, + port)) + return False + + def update_leaders_and_non_leaders(self): + """Get leader node and non-leader nodes of percona. + + Update and set on the object the leader node and list of non-leader + nodes. + + :side effect: sets self.leader and self.non_leaders + :returns: None + """ + status = zaza.model.get_status().applications[self.application] + # Reset + self.leader = None + self.non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + self.leader = unit + else: + self.non_leaders.append(unit) + + +class PerconaClusterCharmTests(PerconaClusterTest): + """Base for percona-cluster charm tests. + + Note: these have tests have been ported from amulet tests + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running percona-cluster tests.""" + super(PerconaClusterTest, cls).setUpClass() + cls.application = "percona-cluster" + cls.services = ["mysqld"] + + def test_100_bootstrapped_and_clustered(self): + """Ensure PXC is bootstrapped and that peer units are clustered.""" + self.units = zaza.model.get_application_config( + self.application)["min-cluster-size"]["value"] + logging.info("Ensuring PXC is bootstrapped") + msg = "Percona cluster failed to bootstrap" + assert self.is_pxc_bootstrapped(), msg + + logging.info("Checking PXC cluster size == {}".format(self.units)) + got = int(self.get_cluster_size()) + msg = ("Percona cluster unexpected size" + " (wanted=%s, got=%s)" % (self.units, got)) + assert got == self.units, msg + + def test_110_restart_on_config_change(self): + """Checking restart happens on config change. + + Change disk format and assert then change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + set_default = {"peer-timeout": "PT3S"} + set_alternate = {"peer-timeout": "PT15S"} + + # Config file affected by juju set config change + conf_file = "/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf" + + # Make config change, check for service restarts + logging.debug("Setting peer timeout ...") + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + {}, {}, + self.services) + logging.info("Passed restart on changed") + + def test_120_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + with self.pause_resume(self.services): + logging.info("Testing pause resume") + + def test_130_change_root_password(self): + """Change root password. + + Change the root password and verify the change was effectively applied. + """ + new_root_passwd = "openstack" + + cmd = ("mysql -uroot -p{} -e\"select 1;\"" + .format(self.get_root_password())) + result = zaza.model.run_on_leader(self.application, cmd) + code = result.get("Code") + output = result.get("Stdout").strip() + + assert code == "0", output + + with self.config_change( + {"root-password": new_root_passwd}, + {"root-password": new_root_passwd}): + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + # try to connect using the new root password + cmd = "mysql -uroot -p{} -e\"select 1;\" ".format(new_root_passwd) + result = zaza.model.run_on_leader(self.application, cmd) + code = result.get("Code") + output = result.get("Stdout").strip() + + assert code == "0", output + + +class PerconaClusterColdStartTest(PerconaClusterTest): + """Percona Cluster cold start tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running percona-cluster cold start tests.""" + super(PerconaClusterColdStartTest, cls).setUpClass() + cls.overcloud_keystone_session = ( + openstack_utils.get_undercloud_keystone_session()) + cls.nova_client = openstack_utils.get_nova_session_client( + cls.overcloud_keystone_session) + cls.machines = ( + juju_utils.get_machine_uuids_for_application(cls.application)) + + def test_100_cold_stop(self): + """Stop all percona-cluster hosts.""" + self.machines.sort() + # Avoid hitting an update-status hook + logging.debug("Wait till model is idle ...") + time.sleep(30) + zaza.model.block_until_all_units_idle() + logging.info("Stopping instances: {}".format(self.machines)) + for uuid in self.machines: + self.nova_client.servers.stop(uuid) + # Unfortunately, juju reports units in workload status "active" + # when they are in fact down. So we have to rely on a simple wait + # and idle check. + logging.debug("Sleep ...") + time.sleep(30) + logging.debug("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + def test_101_cold_start(self): + """Start all percona-cluster hosts in a different order.""" + self.machines.sort(reverse=True) + logging.info("Starting instances: {}".format(self.machines)) + for uuid in self.machines: + self.nova_client.servers.start(uuid) + logging.debug("Sleep ...") + time.sleep(60) + logging.debug("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.debug("Wait for application states ...") + states = {"percona-cluster": { + "workload-status": "blocked", + "workload-status-message": "MySQL is down"}} + zaza.model.wait_for_application_states(states=states) + + def test_102_cold_start_bootstrap(self): + """Bootstrap a non-leader node. + + After bootstrapping a non-leader node, notify bootstrapped on the + leader node. + """ + # Update which node is the leader and which are not + self.update_leaders_and_non_leaders() + # We want to test the worst possible scenario which is the + # non-leader with the highest sequence number. We will use the leader + # for the notify-bootstrapped after. They just need to be different + # units. + logging.info("Execute bootstrap-pxc action after cold boot ...") + zaza.model.run_action( + self.non_leaders[0], + "bootstrap-pxc", + action_params={}) + logging.debug("Wait for application states ...") + states = {"percona-cluster": { + "workload-status": "waiting", + "workload-status-message": "Unit waiting for cluster bootstrap"}} + zaza.model.wait_for_application_states( + states=states) + logging.info("Execute notify-bootstrapped action after cold boot on " + "the leader node ...") + zaza.model.run_action_on_leader( + self.application, + "notify-bootstrapped", + action_params={}) + logging.debug("Wait for application states ...") + test_config = lifecycle_utils.get_charm_config() + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {})) + + +class PerconaClusterScaleTests(PerconaClusterTest): + """Percona Cluster scale tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running percona scale tests. + + Note: these have tests have been ported from amulet tests + """ + super(PerconaClusterScaleTests, cls).setUpClass() + + def test_100_kill_crm_master(self): + """Ensure VIP failover. + + When killing the mysqld on the crm_master unit verify the VIP fails + over. + """ + logging.info("Testing failover of crm_master unit on mysqld failure") + # we are going to kill the crm_master + old_crm_master = self.get_crm_master() + logging.info( + "kill -9 mysqld on {}".format(old_crm_master) + ) + cmd = "sudo killall -9 mysqld" + zaza.model.run_on_unit(old_crm_master, cmd) + + logging.info("looking for the new crm_master") + i = 0 + changed = False + while i < 10 and not changed: + i += 1 + time.sleep(5) # give some time to pacemaker to react + new_crm_master = self.get_crm_master() + + if (new_crm_master and new_crm_master != old_crm_master): + logging.info( + "New crm_master unit detected" + " on {}".format(new_crm_master) + ) + changed = True + + assert changed, "The crm_master didn't change" + + assert self.is_port_open(address=self.vip), "cannot connect to vip" diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 6c8d3d9..88687c3 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -206,13 +206,18 @@ class OpenStackBaseTest(unittest.TestCase): logging.debug('Remote unit timestamp {}'.format(mtime)) with self.config_change(default_config, alternate_config): - logging.debug( - 'Waiting for updates to propagate to {}'.format(config_file)) - model.block_until_oslo_config_entries_match( - self.application_name, - config_file, - alternate_entry, - model_name=self.model_name) + # If this is not an OSLO config file set default_config={} + if alternate_entry: + logging.debug( + 'Waiting for updates to propagate to {}' + .format(config_file)) + model.block_until_oslo_config_entries_match( + self.application_name, + config_file, + alternate_entry, + model_name=self.model_name) + else: + model.block_until_all_units_idle(model_name=self.model_name) # Config update has occured and hooks are idle. Any services should # have been restarted by now: @@ -225,13 +230,17 @@ class OpenStackBaseTest(unittest.TestCase): model_name=self.model_name, pgrep_full=pgrep_full) - logging.debug( - 'Waiting for updates to propagate to '.format(config_file)) - model.block_until_oslo_config_entries_match( - self.application_name, - config_file, - default_entry, - model_name=self.model_name) + # If this is not an OSLO config file set default_config={} + if default_entry: + logging.debug( + 'Waiting for updates to propagate to '.format(config_file)) + model.block_until_oslo_config_entries_match( + self.application_name, + config_file, + default_entry, + model_name=self.model_name) + else: + model.block_until_all_units_idle(model_name=self.model_name) @contextlib.contextmanager def pause_resume(self, services, pgrep_full=False): From 05517b566efa80e440a3824319a68c8eb181ea40 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 17 Jul 2019 08:50:07 -0700 Subject: [PATCH 2/3] Review requests for Alex --- zaza/openstack/charm_tests/mysql/tests.py | 26 +++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index b5d4592..edab7bd 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -57,7 +57,8 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): def get_root_password(self): """Get the MySQL root password. - :returns: str password + :returns: Password + :rtype: str """ return zaza.model.run_on_leader( self.application, @@ -68,7 +69,8 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): :param attr: Attribute to query :type attr: str - :returns: str wsrep value + :returns: wsrep value + :rtype: str """ root_password = self.get_root_password() cmd = ("mysql -uroot -p{} -e\"show status like '{}';\"| " @@ -84,7 +86,8 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): Query the wsrep_ready status in the DB. - :returns: boolean + :returns: True if bootstrapped + :rtype: boolean """ value = self.get_wsrep_value("wsrep_ready") return value.lower() in ["on", "ready"] @@ -94,7 +97,8 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): Query the wsrep_cluster size in the DB. - :returns: str Numeric cluster size + :returns: Numeric cluster size + :rtype: str """ return self.get_wsrep_value("wsrep_cluster_size") @@ -103,7 +107,8 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): Query CRM to determine which node hosts the VIP. - :returns: str Unit name + :returns: Unit name + :rtype: str """ for unit in zaza.model.get_units(self.application): # is the vip running here? @@ -129,7 +134,8 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): :type port: str :param address: IP address :type port: str - :returns: boolean + :returns: True if port is reachable + :rtype: boolean """ try: telnetlib.Telnet(address, port) @@ -149,8 +155,8 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): Update and set on the object the leader node and list of non-leader nodes. - :side effect: sets self.leader and self.non_leaders :returns: None + :rtype: None """ status = zaza.model.get_status().applications[self.application] # Reset @@ -166,7 +172,7 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): class PerconaClusterCharmTests(PerconaClusterTest): """Base for percona-cluster charm tests. - Note: these have tests have been ported from amulet tests + .. note:: these have tests have been ported from amulet tests """ @classmethod @@ -330,6 +336,8 @@ class PerconaClusterColdStartTest(PerconaClusterTest): self.application, "notify-bootstrapped", action_params={}) + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() logging.debug("Wait for application states ...") test_config = lifecycle_utils.get_charm_config() zaza.model.wait_for_application_states( @@ -343,7 +351,7 @@ class PerconaClusterScaleTests(PerconaClusterTest): def setUpClass(cls): """Run class setup for running percona scale tests. - Note: these have tests have been ported from amulet tests + .. note:: these have tests have been ported from amulet tests """ super(PerconaClusterScaleTests, cls).setUpClass() From 81604c399b7d04e80d8a3fea00d34fb2f134c382 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 17 Jul 2019 09:06:06 -0700 Subject: [PATCH 3/3] Review requests for Liam --- .../utilities/test_zaza_utilities_generic.py | 16 ++++ zaza/openstack/charm_tests/mysql/tests.py | 92 +++++++------------ zaza/openstack/utilities/generic.py | 27 ++++++ 3 files changed, 77 insertions(+), 58 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index b50e2b8..4712d75 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -545,3 +545,19 @@ class TestGenericUtils(ut_utils.BaseTestCase): bad_name = 'bad_name' with self.assertRaises(zaza_exceptions.UbuntuReleaseNotFound): generic_utils.get_ubuntu_release(bad_name) + + def test_is_port_open(self): + self.patch( + 'zaza.openstack.utilities.generic.telnetlib.Telnet', + new_callable=mock.MagicMock(), + name='telnet' + ) + + _port = "80" + _addr = "10.5.254.20" + + self.assertTrue(generic_utils.is_port_open(_port, _addr)) + self.telnet.assert_called_with(_addr, _port) + + self.telnet.side_effect = generic_utils.socket.error + self.assertFalse(generic_utils.is_port_open(_port, _addr)) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index edab7bd..6eb11a3 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -17,8 +17,6 @@ import logging import os import re -import socket -import telnetlib import time import zaza.charm_lifecycle.utils as lifecycle_utils @@ -26,6 +24,7 @@ import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.utilities.generic as generic_utils class MySQLTest(test_utils.OpenStackBaseTest): @@ -111,44 +110,19 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): :rtype: str """ for unit in zaza.model.get_units(self.application): - # is the vip running here? - cmd = 'sudo ip a | grep "inet {}/"'.format(self.vip) - result = zaza.model.run_on_unit(unit.entity_id, cmd) - code = result.get("Code") - output = result.get("Stdout").strip() logging.info("Checking {}".format(unit.entity_id)) + # is the vip running here? + cmd = "ip -br addr" + result = zaza.model.run_on_unit(unit.entity_id, cmd) + output = result.get("Stdout").strip() logging.debug(output) - if code == "0": + if self.vip in output: logging.info("vip ({}) running in {}".format( self.vip, unit.entity_id) ) return unit.entity_id - def is_port_open(self, port="3306", address=None): - """Determine if MySQL is accessible. - - Connect to the MySQL port on the VIP. - - :param port: Port number - :type port: str - :param address: IP address - :type port: str - :returns: True if port is reachable - :rtype: boolean - """ - try: - telnetlib.Telnet(address, port) - return True - except socket.error as e: - if e.errno == 113: - self.log.error("could not connect to %s:%s" % (address, port)) - if e.errno == 111: - self.log.error("connection refused connecting" - " to %s:%s" % (address, - port)) - return False - def update_leaders_and_non_leaders(self): """Get leader node and non-leader nodes of percona. @@ -190,11 +164,11 @@ class PerconaClusterCharmTests(PerconaClusterTest): msg = "Percona cluster failed to bootstrap" assert self.is_pxc_bootstrapped(), msg - logging.info("Checking PXC cluster size == {}".format(self.units)) - got = int(self.get_cluster_size()) + logging.info("Checking PXC cluster size >= {}".format(self.units)) + cluster_size = int(self.get_cluster_size()) msg = ("Percona cluster unexpected size" - " (wanted=%s, got=%s)" % (self.units, got)) - assert got == self.units, msg + " (wanted=%s, cluster_size=%s)" % (self.units, cluster_size)) + assert cluster_size >= self.units, msg def test_110_restart_on_config_change(self): """Checking restart happens on config change. @@ -272,12 +246,16 @@ class PerconaClusterColdStartTest(PerconaClusterTest): cls.machines = ( juju_utils.get_machine_uuids_for_application(cls.application)) - def test_100_cold_stop(self): - """Stop all percona-cluster hosts.""" + def test_100_cold_start_bootstrap(self): + """Bootstrap a non-leader node. + + After bootstrapping a non-leader node, notify bootstrapped on the + leader node. + """ + # Stop Nodes self.machines.sort() # Avoid hitting an update-status hook logging.debug("Wait till model is idle ...") - time.sleep(30) zaza.model.block_until_all_units_idle() logging.info("Stopping instances: {}".format(self.machines)) for uuid in self.machines: @@ -290,29 +268,22 @@ class PerconaClusterColdStartTest(PerconaClusterTest): logging.debug("Wait till model is idle ...") zaza.model.block_until_all_units_idle() - def test_101_cold_start(self): - """Start all percona-cluster hosts in a different order.""" + # Start nodes self.machines.sort(reverse=True) logging.info("Starting instances: {}".format(self.machines)) for uuid in self.machines: self.nova_client.servers.start(uuid) - logging.debug("Sleep ...") - time.sleep(60) + logging.debug("Wait till model is idle ...") zaza.model.block_until_all_units_idle() - logging.debug("Wait for application states ...") + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") states = {"percona-cluster": { "workload-status": "blocked", "workload-status-message": "MySQL is down"}} zaza.model.wait_for_application_states(states=states) - def test_102_cold_start_bootstrap(self): - """Bootstrap a non-leader node. - - After bootstrapping a non-leader node, notify bootstrapped on the - leader node. - """ # Update which node is the leader and which are not self.update_leaders_and_non_leaders() # We want to test the worst possible scenario which is the @@ -325,6 +296,8 @@ class PerconaClusterColdStartTest(PerconaClusterTest): "bootstrap-pxc", action_params={}) logging.debug("Wait for application states ...") + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") states = {"percona-cluster": { "workload-status": "waiting", "workload-status-message": "Unit waiting for cluster bootstrap"}} @@ -336,9 +309,9 @@ class PerconaClusterColdStartTest(PerconaClusterTest): self.application, "notify-bootstrapped", action_params={}) - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() logging.debug("Wait for application states ...") + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") test_config = lifecycle_utils.get_charm_config() zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) @@ -372,8 +345,7 @@ class PerconaClusterScaleTests(PerconaClusterTest): logging.info("looking for the new crm_master") i = 0 - changed = False - while i < 10 and not changed: + while i < 10: i += 1 time.sleep(5) # give some time to pacemaker to react new_crm_master = self.get_crm_master() @@ -383,8 +355,12 @@ class PerconaClusterScaleTests(PerconaClusterTest): "New crm_master unit detected" " on {}".format(new_crm_master) ) - changed = True + break + else: + assert False, "The crm_master didn't change" - assert changed, "The crm_master didn't change" - - assert self.is_port_open(address=self.vip), "cannot connect to vip" + # Check connectivity on the VIP + # \ is required due to pep8 and parenthesis would make the assertion + # always true. + assert generic_utils.is_port_open("3306", self.vip), \ + "Cannot connect to vip" diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 50566f7..6990998 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -16,7 +16,9 @@ import logging import os +import socket import subprocess +import telnetlib import yaml from zaza import model @@ -669,3 +671,28 @@ def get_ubuntu_release(ubuntu_name): format(ubuntu_name, UBUNTU_OPENSTACK_RELEASE)) raise zaza_exceptions.UbuntuReleaseNotFound(msg) return index + + +def is_port_open(port, address): + """Determine if TCP port is accessible. + + Connect to the MySQL port on the VIP. + + :param port: Port number + :type port: str + :param address: IP address + :type port: str + :returns: True if port is reachable + :rtype: boolean + """ + try: + telnetlib.Telnet(address, port) + return True + except socket.error as e: + if e.errno == 113: + logging.error("could not connect to {}:{}" + .format(address, port)) + if e.errno == 111: + logging.error("connection refused connecting" + " to {}:{}".format(address, port)) + return False