arif/zaza-openstack-tests
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #30 from thedac/percona-zaza-tests

Browse Source 
Zaza tests for percona-cluster
This commit is contained in:
Liam Young
2019-07-18 12:35:24 +01:00
committed by GitHub
parent 18d928a55e 81604c399b
commit bf2c7bba48
5 changed files with 446 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
unit_tests/utilities/test_zaza_utilities_generic.py
View File

@@ -545,3 +545,19 @@ class TestGenericUtils(ut_utils.BaseTestCase):
bad_name = 'bad_name'
with self.assertRaises(zaza_exceptions.UbuntuReleaseNotFound):
generic_utils.get_ubuntu_release(bad_name)
def test_is_port_open(self):
self.patch(
'zaza.openstack.utilities.generic.telnetlib.Telnet',
new_callable=mock.MagicMock(),
name='telnet'
)
_port = "80"
_addr = "10.5.254.20"
self.assertTrue(generic_utils.is_port_open(_port, _addr))
self.telnet.assert_called_with(_addr, _port)
self.telnet.side_effect = generic_utils.socket.error
self.assertFalse(generic_utils.is_port_open(_port, _addr))

14
zaza/openstack/charm_tests/mysql/__init__.py Normal file
View File

@@ -0,0 +1,14 @@
# Copyright 2019 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Collection of code for setting up and testing mysql or percona-cluster."""

366
zaza/openstack/charm_tests/mysql/tests.py Normal file
View File

@@ -0,0 +1,366 @@
# Copyright 2019 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MySQL/Percona Cluster Testing."""
import logging
import os
import re
import time
import zaza.charm_lifecycle.utils as lifecycle_utils
import zaza.model
import zaza.openstack.charm_tests.test_utils as test_utils
import zaza.openstack.utilities.juju as juju_utils
import zaza.openstack.utilities.openstack as openstack_utils
import zaza.openstack.utilities.generic as generic_utils
class MySQLTest(test_utils.OpenStackBaseTest):
"""Base for mysql charm tests."""
@classmethod
def setUpClass(cls):
"""Run class setup for running mysql tests."""
super(MySQLTest, cls).setUpClass()
cls.application = "mysql"
cls.services = ["mysqld"]
class PerconaClusterTest(test_utils.OpenStackBaseTest):
"""Base for percona-cluster charm tests."""
@classmethod
def setUpClass(cls):
"""Run class setup for running percona-cluster tests."""
super(PerconaClusterTest, cls).setUpClass()
cls.application = "percona-cluster"
# This is the service pidof will attempt to find
# rather than what systemctl uses
cls.services = ["mysqld"]
cls.vip = os.environ.get("OS_VIP00")
cls.leader = None
cls.non_leaders = []
def get_root_password(self):
"""Get the MySQL root password.
:returns: Password
:rtype: str
"""
return zaza.model.run_on_leader(
self.application,
"leader-get root-password")["Stdout"].strip()
def get_wsrep_value(self, attr):
"""Get wsrrep value from the DB.
:param attr: Attribute to query
:type attr: str
:returns: wsrep value
:rtype: str
"""
root_password = self.get_root_password()
cmd = ("mysql -uroot -p{} -e\"show status like '{}';\"| "
"grep {}".format(root_password, attr, attr))
output = zaza.model.run_on_leader(
self.application, cmd)["Stdout"].strip()
value = re.search(r"^.+?\s+(.+)", output).group(1)
logging.debug("%s = %s" % (attr, value))
return value
def is_pxc_bootstrapped(self):
"""Determine if the cluster is bootstrapped.
Query the wsrep_ready status in the DB.
:returns: True if bootstrapped
:rtype: boolean
"""
value = self.get_wsrep_value("wsrep_ready")
return value.lower() in ["on", "ready"]
def get_cluster_size(self):
"""Determine the cluster size.
Query the wsrep_cluster size in the DB.
:returns: Numeric cluster size
:rtype: str
"""
return self.get_wsrep_value("wsrep_cluster_size")
def get_crm_master(self):
"""Determine CRM master for the VIP.
Query CRM to determine which node hosts the VIP.
:returns: Unit name
:rtype: str
"""
for unit in zaza.model.get_units(self.application):
logging.info("Checking {}".format(unit.entity_id))
# is the vip running here?
cmd = "ip -br addr"
result = zaza.model.run_on_unit(unit.entity_id, cmd)
output = result.get("Stdout").strip()
logging.debug(output)
if self.vip in output:
logging.info("vip ({}) running in {}".format(
self.vip,
unit.entity_id)
)
return unit.entity_id
def update_leaders_and_non_leaders(self):
"""Get leader node and non-leader nodes of percona.
Update and set on the object the leader node and list of non-leader
nodes.
:returns: None
:rtype: None
"""
status = zaza.model.get_status().applications[self.application]
# Reset
self.leader = None
self.non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
self.leader = unit
else:
self.non_leaders.append(unit)
class PerconaClusterCharmTests(PerconaClusterTest):
"""Base for percona-cluster charm tests.
.. note:: these have tests have been ported from amulet tests
"""
@classmethod
def setUpClass(cls):
"""Run class setup for running percona-cluster tests."""
super(PerconaClusterTest, cls).setUpClass()
cls.application = "percona-cluster"
cls.services = ["mysqld"]
def test_100_bootstrapped_and_clustered(self):
"""Ensure PXC is bootstrapped and that peer units are clustered."""
self.units = zaza.model.get_application_config(
self.application)["min-cluster-size"]["value"]
logging.info("Ensuring PXC is bootstrapped")
msg = "Percona cluster failed to bootstrap"
assert self.is_pxc_bootstrapped(), msg
logging.info("Checking PXC cluster size >= {}".format(self.units))
cluster_size = int(self.get_cluster_size())
msg = ("Percona cluster unexpected size"
" (wanted=%s, cluster_size=%s)" % (self.units, cluster_size))
assert cluster_size >= self.units, msg
def test_110_restart_on_config_change(self):
"""Checking restart happens on config change.
Change disk format and assert then change propagates to the correct
file and that services are restarted as a result
"""
# Expected default and alternate values
set_default = {"peer-timeout": "PT3S"}
set_alternate = {"peer-timeout": "PT15S"}
# Config file affected by juju set config change
conf_file = "/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf"
# Make config change, check for service restarts
logging.debug("Setting peer timeout ...")
self.restart_on_changed(
conf_file,
set_default,
set_alternate,
{}, {},
self.services)
logging.info("Passed restart on changed")
def test_120_pause_resume(self):
"""Run pause and resume tests.
Pause service and check services are stopped then resume and check
they are started
"""
with self.pause_resume(self.services):
logging.info("Testing pause resume")
def test_130_change_root_password(self):
"""Change root password.
Change the root password and verify the change was effectively applied.
"""
new_root_passwd = "openstack"
cmd = ("mysql -uroot -p{} -e\"select 1;\""
.format(self.get_root_password()))
result = zaza.model.run_on_leader(self.application, cmd)
code = result.get("Code")
output = result.get("Stdout").strip()
assert code == "0", output
with self.config_change(
{"root-password": new_root_passwd},
{"root-password": new_root_passwd}):
logging.info("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
# try to connect using the new root password
cmd = "mysql -uroot -p{} -e\"select 1;\" ".format(new_root_passwd)
result = zaza.model.run_on_leader(self.application, cmd)
code = result.get("Code")
output = result.get("Stdout").strip()
assert code == "0", output
class PerconaClusterColdStartTest(PerconaClusterTest):
"""Percona Cluster cold start tests."""
@classmethod
def setUpClass(cls):
"""Run class setup for running percona-cluster cold start tests."""
super(PerconaClusterColdStartTest, cls).setUpClass()
cls.overcloud_keystone_session = (
openstack_utils.get_undercloud_keystone_session())
cls.nova_client = openstack_utils.get_nova_session_client(
cls.overcloud_keystone_session)
cls.machines = (
juju_utils.get_machine_uuids_for_application(cls.application))
def test_100_cold_start_bootstrap(self):
"""Bootstrap a non-leader node.
After bootstrapping a non-leader node, notify bootstrapped on the
leader node.
"""
# Stop Nodes
self.machines.sort()
# Avoid hitting an update-status hook
logging.debug("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
logging.info("Stopping instances: {}".format(self.machines))
for uuid in self.machines:
self.nova_client.servers.stop(uuid)
# Unfortunately, juju reports units in workload status "active"
# when they are in fact down. So we have to rely on a simple wait
# and idle check.
logging.debug("Sleep ...")
time.sleep(30)
logging.debug("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
# Start nodes
self.machines.sort(reverse=True)
logging.info("Starting instances: {}".format(self.machines))
for uuid in self.machines:
self.nova_client.servers.start(uuid)
logging.debug("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
logging.debug("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
states = {"percona-cluster": {
"workload-status": "blocked",
"workload-status-message": "MySQL is down"}}
zaza.model.wait_for_application_states(states=states)
# Update which node is the leader and which are not
self.update_leaders_and_non_leaders()
# We want to test the worst possible scenario which is the
# non-leader with the highest sequence number. We will use the leader
# for the notify-bootstrapped after. They just need to be different
# units.
logging.info("Execute bootstrap-pxc action after cold boot ...")
zaza.model.run_action(
self.non_leaders[0],
"bootstrap-pxc",
action_params={})
logging.debug("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
states = {"percona-cluster": {
"workload-status": "waiting",
"workload-status-message": "Unit waiting for cluster bootstrap"}}
zaza.model.wait_for_application_states(
states=states)
logging.info("Execute notify-bootstrapped action after cold boot on "
"the leader node ...")
zaza.model.run_action_on_leader(
self.application,
"notify-bootstrapped",
action_params={})
logging.debug("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
test_config = lifecycle_utils.get_charm_config()
zaza.model.wait_for_application_states(
states=test_config.get("target_deploy_status", {}))
class PerconaClusterScaleTests(PerconaClusterTest):
"""Percona Cluster scale tests."""
@classmethod
def setUpClass(cls):
"""Run class setup for running percona scale tests.
.. note:: these have tests have been ported from amulet tests
"""
super(PerconaClusterScaleTests, cls).setUpClass()
def test_100_kill_crm_master(self):
"""Ensure VIP failover.
When killing the mysqld on the crm_master unit verify the VIP fails
over.
"""
logging.info("Testing failover of crm_master unit on mysqld failure")
# we are going to kill the crm_master
old_crm_master = self.get_crm_master()
logging.info(
"kill -9 mysqld on {}".format(old_crm_master)
)
cmd = "sudo killall -9 mysqld"
zaza.model.run_on_unit(old_crm_master, cmd)
logging.info("looking for the new crm_master")
i = 0
while i < 10:
i += 1
time.sleep(5) # give some time to pacemaker to react
new_crm_master = self.get_crm_master()
if (new_crm_master and new_crm_master != old_crm_master):
logging.info(
"New crm_master unit detected"
" on {}".format(new_crm_master)
)
break
else:
assert False, "The crm_master didn't change"
# Check connectivity on the VIP
# \ is required due to pep8 and parenthesis would make the assertion
# always true.
assert generic_utils.is_port_open("3306", self.vip), \
"Cannot connect to vip"

37
zaza/openstack/charm_tests/test_utils.py
View File

@@ -206,13 +206,18 @@ class OpenStackBaseTest(unittest.TestCase):
logging.debug('Remote unit timestamp {}'.format(mtime))
with self.config_change(default_config, alternate_config):
logging.debug(
'Waiting for updates to propagate to {}'.format(config_file))
model.block_until_oslo_config_entries_match(
self.application_name,
config_file,
alternate_entry,
model_name=self.model_name)
# If this is not an OSLO config file set default_config={}
if alternate_entry:
logging.debug(
'Waiting for updates to propagate to {}'
.format(config_file))
model.block_until_oslo_config_entries_match(
self.application_name,
config_file,
alternate_entry,
model_name=self.model_name)
else:
model.block_until_all_units_idle(model_name=self.model_name)
# Config update has occured and hooks are idle. Any services should
# have been restarted by now:
@@ -225,13 +230,17 @@ class OpenStackBaseTest(unittest.TestCase):
model_name=self.model_name,
pgrep_full=pgrep_full)
logging.debug(
'Waiting for updates to propagate to '.format(config_file))
model.block_until_oslo_config_entries_match(
self.application_name,
config_file,
default_entry,
model_name=self.model_name)
# If this is not an OSLO config file set default_config={}
if default_entry:
logging.debug(
'Waiting for updates to propagate to '.format(config_file))
model.block_until_oslo_config_entries_match(
self.application_name,
config_file,
default_entry,
model_name=self.model_name)
else:
model.block_until_all_units_idle(model_name=self.model_name)
@contextlib.contextmanager
def pause_resume(self, services, pgrep_full=False):

27
zaza/openstack/utilities/generic.py
View File

@@ -16,7 +16,9 @@
import logging
import os
import socket
import subprocess
import telnetlib
import yaml
from zaza import model
@@ -669,3 +671,28 @@ def get_ubuntu_release(ubuntu_name):
format(ubuntu_name, UBUNTU_OPENSTACK_RELEASE))
raise zaza_exceptions.UbuntuReleaseNotFound(msg)
return index
def is_port_open(port, address):
"""Determine if TCP port is accessible.
Connect to the MySQL port on the VIP.
:param port: Port number
:type port: str
:param address: IP address
:type port: str
:returns: True if port is reachable
:rtype: boolean
"""
try:
telnetlib.Telnet(address, port)
return True
except socket.error as e:
if e.errno == 113:
logging.error("could not connect to {}:{}"
.format(address, port))
if e.errno == 111:
logging.error("connection refused connecting"
" to {}:{}".format(address, port))
return False