553 lines
20 KiB
Python
553 lines
20 KiB
Python
# Copyright 2019 Canonical Ltd.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""MySQL/Percona Cluster Testing."""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import time
|
|
|
|
import zaza.charm_lifecycle.utils as lifecycle_utils
|
|
import zaza.model
|
|
import zaza.openstack.charm_tests.test_utils as test_utils
|
|
import zaza.openstack.utilities.juju as juju_utils
|
|
import zaza.openstack.utilities.openstack as openstack_utils
|
|
import zaza.openstack.utilities.generic as generic_utils
|
|
|
|
|
|
class MySQLBaseTest(test_utils.OpenStackBaseTest):
|
|
"""Base for mysql charm tests."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Run class setup for running mysql tests."""
|
|
super(MySQLBaseTest, cls).setUpClass()
|
|
cls.application = "mysql"
|
|
cls.services = ["mysqld"]
|
|
# Config file affected by juju set config change
|
|
cls.conf_file = "/etc/mysql/mysql.conf.d/mysqld.cnf"
|
|
|
|
def get_root_password(self):
|
|
"""Get the MySQL root password.
|
|
|
|
:returns: Password
|
|
:rtype: str
|
|
"""
|
|
return zaza.model.run_on_leader(
|
|
self.application,
|
|
"leader-get root-password")["Stdout"].strip()
|
|
|
|
def get_leaders_and_non_leaders(self):
|
|
"""Get leader node and non-leader nodes of percona.
|
|
|
|
Update and set on the object the leader node and list of non-leader
|
|
nodes.
|
|
|
|
:returns: None
|
|
:rtype: None
|
|
"""
|
|
status = zaza.model.get_status().applications[self.application]
|
|
# Reset
|
|
self.leader = None
|
|
self.non_leaders = []
|
|
for unit in status["units"]:
|
|
if status["units"][unit].get("leader"):
|
|
self.leader = unit
|
|
else:
|
|
self.non_leaders.append(unit)
|
|
return self.leader, self.non_leaders
|
|
|
|
|
|
class MySQLCommonTests(MySQLBaseTest):
|
|
"""Common mysql charm tests."""
|
|
|
|
def test_910_restart_on_config_change(self):
|
|
"""Checking restart happens on config change.
|
|
|
|
Change disk format and assert then change propagates to the correct
|
|
file and that services are restarted as a result
|
|
"""
|
|
# Expected default and alternate values
|
|
set_default = {"max-connections": "600"}
|
|
set_alternate = {"max-connections": "1000"}
|
|
|
|
# Make config change, check for service restarts
|
|
logging.debug("Setting peer timeout ...")
|
|
self.restart_on_changed(
|
|
self.conf_file,
|
|
set_default,
|
|
set_alternate,
|
|
{}, {},
|
|
self.services)
|
|
logging.info("Passed restart on changed test.")
|
|
|
|
def test_920_pause_resume(self):
|
|
"""Run pause and resume tests.
|
|
|
|
Pause service and check services are stopped then resume and check
|
|
they are started
|
|
"""
|
|
with self.pause_resume(self.services):
|
|
logging.info("Testing pause resume")
|
|
logging.info("Passed pause and resume test.")
|
|
|
|
|
|
class PerconaClusterBaseTest(MySQLBaseTest):
|
|
"""Base for percona-cluster charm tests."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Run class setup for running percona-cluster tests."""
|
|
super().setUpClass()
|
|
cls.application = "percona-cluster"
|
|
# This is the service pidof will attempt to find
|
|
# rather than what systemctl uses
|
|
cls.services = ["mysqld"]
|
|
cls.vip = os.environ.get("TEST_VIP00")
|
|
# Config file affected by juju set config change
|
|
cls.conf_file = "/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf"
|
|
|
|
def get_wsrep_value(self, attr):
|
|
"""Get wsrrep value from the DB.
|
|
|
|
:param attr: Attribute to query
|
|
:type attr: str
|
|
:returns: wsrep value
|
|
:rtype: str
|
|
"""
|
|
root_password = self.get_root_password()
|
|
cmd = ("mysql -uroot -p{} -e\"show status like '{}';\"| "
|
|
"grep {}".format(root_password, attr, attr))
|
|
output = zaza.model.run_on_leader(
|
|
self.application, cmd)["Stdout"].strip()
|
|
value = re.search(r"^.+?\s+(.+)", output).group(1)
|
|
logging.debug("%s = %s" % (attr, value))
|
|
return value
|
|
|
|
def is_pxc_bootstrapped(self):
|
|
"""Determine if the cluster is bootstrapped.
|
|
|
|
Query the wsrep_ready status in the DB.
|
|
|
|
:returns: True if bootstrapped
|
|
:rtype: boolean
|
|
"""
|
|
value = self.get_wsrep_value("wsrep_ready")
|
|
return value.lower() in ["on", "ready"]
|
|
|
|
def get_cluster_size(self):
|
|
"""Determine the cluster size.
|
|
|
|
Query the wsrep_cluster size in the DB.
|
|
|
|
:returns: Numeric cluster size
|
|
:rtype: str
|
|
"""
|
|
return self.get_wsrep_value("wsrep_cluster_size")
|
|
|
|
def get_crm_master(self):
|
|
"""Determine CRM master for the VIP.
|
|
|
|
Query CRM to determine which node hosts the VIP.
|
|
|
|
:returns: Unit name
|
|
:rtype: str
|
|
"""
|
|
for unit in zaza.model.get_units(self.application):
|
|
logging.info("Checking {}".format(unit.entity_id))
|
|
# is the vip running here?
|
|
cmd = "ip -br addr"
|
|
result = zaza.model.run_on_unit(unit.entity_id, cmd)
|
|
output = result.get("Stdout").strip()
|
|
logging.debug(output)
|
|
if self.vip in output:
|
|
logging.info("vip ({}) running in {}".format(
|
|
self.vip,
|
|
unit.entity_id)
|
|
)
|
|
return unit.entity_id
|
|
|
|
|
|
class PerconaClusterCharmTests(MySQLCommonTests, PerconaClusterBaseTest):
|
|
"""Percona-cluster charm tests.
|
|
|
|
.. note:: these have tests have been ported from amulet tests
|
|
"""
|
|
|
|
def test_100_bootstrapped_and_clustered(self):
|
|
"""Ensure PXC is bootstrapped and that peer units are clustered."""
|
|
self.units = zaza.model.get_application_config(
|
|
self.application)["min-cluster-size"]["value"]
|
|
logging.info("Ensuring PXC is bootstrapped")
|
|
msg = "Percona cluster failed to bootstrap"
|
|
assert self.is_pxc_bootstrapped(), msg
|
|
|
|
logging.info("Checking PXC cluster size >= {}".format(self.units))
|
|
cluster_size = int(self.get_cluster_size())
|
|
msg = ("Percona cluster unexpected size"
|
|
" (wanted=%s, cluster_size=%s)" % (self.units, cluster_size))
|
|
assert cluster_size >= self.units, msg
|
|
|
|
def test_130_change_root_password(self):
|
|
"""Change root password.
|
|
|
|
Change the root password and verify the change was effectively applied.
|
|
"""
|
|
new_root_passwd = "openstack"
|
|
|
|
cmd = ("mysql -uroot -p{} -e\"select 1;\""
|
|
.format(self.get_root_password()))
|
|
result = zaza.model.run_on_leader(self.application, cmd)
|
|
code = result.get("Code")
|
|
output = result.get("Stdout").strip()
|
|
|
|
assert code == "0", output
|
|
|
|
with self.config_change(
|
|
{"root-password": new_root_passwd},
|
|
{"root-password": new_root_passwd}):
|
|
|
|
logging.info("Wait till model is idle ...")
|
|
zaza.model.block_until_all_units_idle()
|
|
# try to connect using the new root password
|
|
cmd = "mysql -uroot -p{} -e\"select 1;\" ".format(new_root_passwd)
|
|
result = zaza.model.run_on_leader(self.application, cmd)
|
|
code = result.get("Code")
|
|
output = result.get("Stdout").strip()
|
|
|
|
assert code == "0", output
|
|
|
|
|
|
class PerconaClusterColdStartTest(PerconaClusterBaseTest):
|
|
"""Percona Cluster cold start tests."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Run class setup for running percona-cluster cold start tests."""
|
|
super(PerconaClusterColdStartTest, cls).setUpClass()
|
|
cls.overcloud_keystone_session = (
|
|
openstack_utils.get_undercloud_keystone_session())
|
|
cls.nova_client = openstack_utils.get_nova_session_client(
|
|
cls.overcloud_keystone_session)
|
|
|
|
def resolve_update_status_errors(self):
|
|
"""Resolve update-status hooks error.
|
|
|
|
This should *only* be used after an instance hard reboot to handle the
|
|
situation where a update-status hook was running when the unit was
|
|
rebooted.
|
|
"""
|
|
zaza.model.resolve_units(
|
|
application_name='percona-cluster',
|
|
erred_hook='update-status',
|
|
wait=True)
|
|
zaza.model.resolve_units(
|
|
application_name='hacluster',
|
|
erred_hook='update-status',
|
|
wait=True)
|
|
|
|
def test_100_cold_start_bootstrap(self):
|
|
"""Bootstrap a non-leader node.
|
|
|
|
After bootstrapping a non-leader node, notify bootstrapped on the
|
|
leader node.
|
|
"""
|
|
_machines = list(
|
|
juju_utils.get_machine_uuids_for_application(self.application))
|
|
# Stop Nodes
|
|
_machines.sort()
|
|
# Avoid hitting an update-status hook
|
|
logging.debug("Wait till model is idle ...")
|
|
zaza.model.block_until_all_units_idle()
|
|
logging.info("Stopping instances: {}".format(_machines))
|
|
for uuid in _machines:
|
|
self.nova_client.servers.stop(uuid)
|
|
logging.debug("Wait till all machines are shutoff ...")
|
|
for uuid in _machines:
|
|
openstack_utils.resource_reaches_status(self.nova_client.servers,
|
|
uuid,
|
|
expected_status='SHUTOFF',
|
|
stop_after_attempt=16)
|
|
|
|
# Start nodes
|
|
_machines.sort(reverse=True)
|
|
logging.info("Starting instances: {}".format(_machines))
|
|
for uuid in _machines:
|
|
self.nova_client.servers.start(uuid)
|
|
|
|
for unit in zaza.model.get_units(self.application):
|
|
zaza.model.block_until_unit_wl_status(
|
|
unit.entity_id,
|
|
'unknown',
|
|
negate_match=True)
|
|
|
|
logging.debug("Wait till model is idle ...")
|
|
# XXX If a hook was executing on a unit when it was powered off
|
|
# it comes back in an error state.
|
|
try:
|
|
zaza.model.block_until_all_units_idle()
|
|
except zaza.model.UnitError:
|
|
self.resolve_update_status_errors()
|
|
zaza.model.block_until_all_units_idle()
|
|
|
|
logging.debug("Wait for application states ...")
|
|
for unit in zaza.model.get_units(self.application):
|
|
try:
|
|
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
|
|
except zaza.model.UnitError:
|
|
self.resolve_update_status_errors()
|
|
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
|
|
states = {"percona-cluster": {
|
|
"workload-status": "blocked",
|
|
"workload-status-message": "MySQL is down"}}
|
|
zaza.model.wait_for_application_states(states=states)
|
|
|
|
# Update which node is the leader and which are not
|
|
_leader, _non_leaders = self.get_leaders_and_non_leaders()
|
|
# We want to test the worst possible scenario which is the
|
|
# non-leader with the highest sequence number. We will use the leader
|
|
# for the notify-bootstrapped after. They just need to be different
|
|
# units.
|
|
logging.info("Execute bootstrap-pxc action after cold boot ...")
|
|
zaza.model.run_action(
|
|
_non_leaders[0],
|
|
"bootstrap-pxc",
|
|
action_params={})
|
|
logging.debug("Wait for application states ...")
|
|
for unit in zaza.model.get_units(self.application):
|
|
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
|
|
states = {"percona-cluster": {
|
|
"workload-status": "waiting",
|
|
"workload-status-message": "Unit waiting for cluster bootstrap"}}
|
|
zaza.model.wait_for_application_states(
|
|
states=states)
|
|
logging.info("Execute notify-bootstrapped action after cold boot on "
|
|
"the leader node ...")
|
|
zaza.model.run_action_on_leader(
|
|
self.application,
|
|
"notify-bootstrapped",
|
|
action_params={})
|
|
logging.debug("Wait for application states ...")
|
|
for unit in zaza.model.get_units(self.application):
|
|
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
|
|
test_config = lifecycle_utils.get_charm_config(fatal=False)
|
|
zaza.model.wait_for_application_states(
|
|
states=test_config.get("target_deploy_status", {}))
|
|
|
|
|
|
class PerconaClusterScaleTests(PerconaClusterBaseTest):
|
|
"""Percona Cluster scale tests."""
|
|
|
|
def test_100_kill_crm_master(self):
|
|
"""Ensure VIP failover.
|
|
|
|
When killing the mysqld on the crm_master unit verify the VIP fails
|
|
over.
|
|
"""
|
|
logging.info("Testing failover of crm_master unit on mysqld failure")
|
|
# we are going to kill the crm_master
|
|
old_crm_master = self.get_crm_master()
|
|
logging.info(
|
|
"kill -9 mysqld on {}".format(old_crm_master)
|
|
)
|
|
cmd = "sudo killall -9 mysqld"
|
|
zaza.model.run_on_unit(old_crm_master, cmd)
|
|
|
|
logging.info("looking for the new crm_master")
|
|
i = 0
|
|
while i < 10:
|
|
i += 1
|
|
# XXX time.sleep roundup
|
|
# https://github.com/openstack-charmers/zaza-openstack-tests/issues/46
|
|
time.sleep(5) # give some time to pacemaker to react
|
|
new_crm_master = self.get_crm_master()
|
|
|
|
if (new_crm_master and new_crm_master != old_crm_master):
|
|
logging.info(
|
|
"New crm_master unit detected"
|
|
" on {}".format(new_crm_master)
|
|
)
|
|
break
|
|
else:
|
|
assert False, "The crm_master didn't change"
|
|
|
|
# Check connectivity on the VIP
|
|
# \ is required due to pep8 and parenthesis would make the assertion
|
|
# always true.
|
|
assert generic_utils.is_port_open("3306", self.vip), \
|
|
"Cannot connect to vip"
|
|
|
|
|
|
class MySQLInnoDBClusterTests(MySQLCommonTests):
|
|
"""Mysql-innodb-cluster charm tests.
|
|
|
|
Note: The restart on changed and pause/resume tests also validate the
|
|
changing of the R/W primary. On each mysqld shutodown a new R/W primary is
|
|
elected automatically by MySQL.
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Run class setup for running mysql-innodb-cluster tests."""
|
|
super().setUpClass()
|
|
cls.application = "mysql-innodb-cluster"
|
|
|
|
def test_100_cluster_status(self):
|
|
"""Checking cluster status.
|
|
|
|
Run the cluster-status action.
|
|
"""
|
|
logging.info("Execute cluster-status action")
|
|
action = zaza.model.run_action_on_leader(
|
|
self.application,
|
|
"cluster-status",
|
|
action_params={})
|
|
cluster_status = json.loads(action.data["results"]["cluster-status"])
|
|
assert "OK" in cluster_status["defaultReplicaSet"]["status"], (
|
|
"Cluster status action failed: {}"
|
|
.format(action.data))
|
|
logging.info("Passed cluster-status action test.")
|
|
|
|
def test_110_mysqldump(self):
|
|
"""Backup mysql.
|
|
|
|
Run the mysqldump action.
|
|
"""
|
|
_db = "keystone"
|
|
_file_key = "mysqldump-file"
|
|
logging.info("Execute mysqldump action")
|
|
action = zaza.model.run_action_on_leader(
|
|
self.application,
|
|
"mysqldump",
|
|
action_params={"databases": _db})
|
|
_results = action.data["results"]
|
|
assert _db in _results[_file_key], (
|
|
"Mysqldump action failed: {}".format(action.data))
|
|
logging.info("Passed mysqldump action test.")
|
|
|
|
def test_120_set_cluster_option(self):
|
|
"""Set cluster option.
|
|
|
|
Run the set-cluster-option action.
|
|
"""
|
|
_key = "autoRejoinTries"
|
|
_value = "500"
|
|
logging.info("Set cluster option {}={}".format(_key, _value))
|
|
action = zaza.model.run_action_on_leader(
|
|
self.application,
|
|
"set-cluster-option",
|
|
action_params={"key": _key, "value": _value})
|
|
assert "Success" in action.data["results"]["outcome"], (
|
|
"Set cluster option {}={} action failed: {}"
|
|
.format(_key, _value, action.data))
|
|
logging.info("Passed set cluster option action test.")
|
|
|
|
|
|
class MySQLInnoDBClusterColdStartTest(MySQLBaseTest):
|
|
"""Percona Cluster cold start tests."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Run class setup for running percona-cluster cold start tests."""
|
|
super().setUpClass()
|
|
cls.application = "mysql-innodb-cluster"
|
|
cls.overcloud_keystone_session = (
|
|
openstack_utils.get_undercloud_keystone_session())
|
|
cls.nova_client = openstack_utils.get_nova_session_client(
|
|
cls.overcloud_keystone_session)
|
|
|
|
def resolve_update_status_errors(self):
|
|
"""Resolve update-status hooks error.
|
|
|
|
This should *only* be used after an instance hard reboot to handle the
|
|
situation where a update-status hook was running when the unit was
|
|
rebooted.
|
|
"""
|
|
zaza.model.resolve_units(
|
|
application_name=self.application,
|
|
erred_hook='update-status',
|
|
wait=True)
|
|
|
|
def test_100_reboot_cluster_from_complete_outage(self):
|
|
"""Reboot cluster from complete outage.
|
|
|
|
After a cold start, reboot cluster from complete outage.
|
|
"""
|
|
_machines = list(
|
|
juju_utils.get_machine_uuids_for_application(self.application))
|
|
# Stop Nodes
|
|
_machines.sort()
|
|
# Avoid hitting an update-status hook
|
|
logging.debug("Wait till model is idle ...")
|
|
zaza.model.block_until_all_units_idle()
|
|
logging.info("Stopping instances: {}".format(_machines))
|
|
for uuid in _machines:
|
|
self.nova_client.servers.stop(uuid)
|
|
logging.debug("Wait till all machines are shutoff ...")
|
|
for uuid in _machines:
|
|
openstack_utils.resource_reaches_status(self.nova_client.servers,
|
|
uuid,
|
|
expected_status='SHUTOFF',
|
|
stop_after_attempt=16)
|
|
|
|
# Start nodes
|
|
_machines.sort(reverse=True)
|
|
logging.info("Starting instances: {}".format(_machines))
|
|
for uuid in _machines:
|
|
self.nova_client.servers.start(uuid)
|
|
|
|
for unit in zaza.model.get_units(self.application):
|
|
zaza.model.block_until_unit_wl_status(
|
|
unit.entity_id,
|
|
'unknown',
|
|
negate_match=True)
|
|
|
|
logging.debug("Wait till model is idle ...")
|
|
try:
|
|
zaza.model.block_until_all_units_idle()
|
|
except zaza.model.UnitError:
|
|
self.resolve_update_status_errors()
|
|
zaza.model.block_until_all_units_idle()
|
|
|
|
logging.debug("Wait for application states ...")
|
|
for unit in zaza.model.get_units(self.application):
|
|
try:
|
|
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
|
|
except zaza.model.UnitError:
|
|
self.resolve_update_status_errors()
|
|
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
|
|
states = {self.application: {
|
|
"workload-status": "blocked",
|
|
"workload-status-message":
|
|
"MySQL InnoDB Cluster not healthy: None"}}
|
|
zaza.model.wait_for_application_states(states=states)
|
|
|
|
logging.info("Execute reboot-cluster-from-complete-outage "
|
|
"action after cold boot ...")
|
|
action = zaza.model.run_action_on_leader(
|
|
self.application,
|
|
"reboot-cluster-from-complete-outage",
|
|
action_params={})
|
|
assert "Success" in action.data["results"]["outcome"], (
|
|
"Reboot cluster from complete outage action failed: {}"
|
|
.format(action.data))
|
|
logging.debug("Wait for application states ...")
|
|
for unit in zaza.model.get_units(self.application):
|
|
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
|
|
test_config = lifecycle_utils.get_charm_config(fatal=False)
|
|
zaza.model.wait_for_application_states(
|
|
states=test_config.get("target_deploy_status", {}))
|