From e16406116b3e9ae8ec7dc31443e6451b62a3f489 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 22 Mar 2023 13:02:38 +0000 Subject: [PATCH 1/7] Add test to rotate mysql service user password This new test verifies that keystone can have its password rotated and then still operate afterwards. It verifies that the on-disk password is changed in the keystone application and that the user list can be performed. --- zaza/openstack/charm_tests/mysql/tests.py | 96 +++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index eb379ff..c78f505 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -14,6 +14,7 @@ """MySQL/Percona Cluster Testing.""" +import configparser import json import logging import os @@ -567,6 +568,101 @@ class MySQLInnoDBClusterTests(MySQLCommonTests): ) +class MySQLInnoDBClusterRotatePasswordTests(MySQLCommonTests): + """Mysql-innodb-cluster charm tests. + + Note: The restart on changed and pause/resume tests also validate the + changing of the R/W primary. On each mysqld shutodown a new R/W primary is + elected automatically by MySQL. + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running mysql-innodb-cluster tests.""" + super().setUpClass() + cls.application = "mysql-innodb-cluster" + + def test_rotate_keystone_service_user_password(self): + """Verify action used to rotate a service user (keystone) password.""" + KEYSTONE_APP = 'keystone' + KEYSTONE_PASSWD_KEY = "mysql-keystone.passwd" + KEYSTONE_CONF_FILE = '/etc/keystone/keystone.conf' + + def _get_password_from_keystone_leader(): + conf = zaza.model.file_contents( + 'keystone/leader', KEYSTONE_CONF_FILE) + config = configparser.ConfigParser() + config.read_string(conf) + connection_info = config['database']['connection'] + match = re.match(r"^mysql.*keystone:(.+)@", connection_info) + if match: + return match[1] + self.fail("Couldn't find mysql password in {}" + .format(connection_info)) + + # only do the test if keystone is in the model + applications = zaza.model.sync_deployed(self.model_name) + if KEYSTONE_APP not in applications: + self.skipTest( + '{} is not deployed, so not doing password rotation' + .format(KEYSTONE_APP)) + + # get the users via the 'list-service-usernames' action. + logging.info( + "Getting usernames from mysql that can have password rotated.") + action = zaza.model.run_action_on_leader( + self.application, + 'list-service-usernames', + action_params={} + ) + usernames = action.data['results']['usernames'] + self.assertIn('keystone', usernames) + logging.info("... usernames: %s", ', '.join(usernames)) + + # grab the password for keystone from the leader / to verify the change + old_keystone_passwd_on_mysql = juju_utils.leader_get( + self.application_name, KEYSTONE_PASSWD_KEY).strip() + old_keystone_passwd_conf = _get_password_from_keystone_leader() + + # verify that keystone is working. + admin_keystone_session = ( + openstack_utils.get_overcloud_keystone_session()) + keystone_client = openstack_utils.get_keystone_session_client( + admin_keystone_session) + keystone_client.users.list() + + # now rotate the password for keystone + # run the action to rotate the password. + logging.info("Rotating password for keystone in mysql.") + zaza.model.run_action_on_leader( + self.application_name, + 'rotate-service-user-password', + action_params={'service-user': 'keystone'}, + ) + + # let everything settle. + logging.info("Waiting for model to settle.") + zaza.model.block_until_all_units_idle() + + # verify that the password has changed. + new_keystone_passwd_on_mysql = juju_utils.leader_get( + self.application_name, KEYSTONE_PASSWD_KEY).strip() + new_keystone_passwd_conf = _get_password_from_keystone_leader() + self.assertNotEqual(old_keystone_passwd_on_mysql, + new_keystone_passwd_on_mysql) + self.assertNotEqual(old_keystone_passwd_conf, + new_keystone_passwd_conf) + self.assertEqual(new_keystone_passwd_on_mysql, + new_keystone_passwd_conf) + + # finally, verify that keystone is still working. + admin_keystone_session = ( + openstack_utils.get_overcloud_keystone_session()) + keystone_client = openstack_utils.get_keystone_session_client( + admin_keystone_session) + keystone_client.users.list() + + class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): """Percona Cluster cold start tests.""" From 32f2a052b1fe44793946f451c44eb9e76cd3e524 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 24 Mar 2023 12:23:28 +0000 Subject: [PATCH 2/7] Add wait_for_agent_status in 'settle' code This is to guarantee that the block check didn't happen prior to the action being started. --- zaza/openstack/charm_tests/mysql/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index c78f505..8039338 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -642,6 +642,7 @@ class MySQLInnoDBClusterRotatePasswordTests(MySQLCommonTests): # let everything settle. logging.info("Waiting for model to settle.") + zaza.model.wait_for_agent_status() zaza.model.block_until_all_units_idle() # verify that the password has changed. From 140805d321305d047deaad43a725e4c9bec40dff Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 29 Mar 2023 12:24:35 +0100 Subject: [PATCH 3/7] Fix handling of usernames in password rotation mysql test This patch ensures that the usernames are parsed correctly from the action to ensure that the usernames are handled correctly. --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 8039338..e7ab003 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -615,7 +615,7 @@ class MySQLInnoDBClusterRotatePasswordTests(MySQLCommonTests): 'list-service-usernames', action_params={} ) - usernames = action.data['results']['usernames'] + usernames = action.data['results']['usernames'].split(',') self.assertIn('keystone', usernames) logging.info("... usernames: %s", ', '.join(usernames)) From a61653d031382899506ce48f1c2123936a001ec1 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 5 Apr 2023 19:12:54 +0100 Subject: [PATCH 4/7] Add a tenacity retry to the password change checks --- zaza/openstack/charm_tests/mysql/tests.py | 31 ++++++++++++++++------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index e7ab003..2ed7f0f 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -646,15 +646,28 @@ class MySQLInnoDBClusterRotatePasswordTests(MySQLCommonTests): zaza.model.block_until_all_units_idle() # verify that the password has changed. - new_keystone_passwd_on_mysql = juju_utils.leader_get( - self.application_name, KEYSTONE_PASSWD_KEY).strip() - new_keystone_passwd_conf = _get_password_from_keystone_leader() - self.assertNotEqual(old_keystone_passwd_on_mysql, - new_keystone_passwd_on_mysql) - self.assertNotEqual(old_keystone_passwd_conf, - new_keystone_passwd_conf) - self.assertEqual(new_keystone_passwd_on_mysql, - new_keystone_passwd_conf) + # Due to the async-ness of the whole model and when the various hooks + # will fire between mysql-innodb-cluster, the mysql-router and + # keystone, so we retry a reasonable time to wait for everything to + # propagate through. + for attempt in tenacity.Retrying( + reraise=True, + wait=tenacity.wait_fixed(30), + stop=tenacity.stop_after_attempt(20), # wait for max 10m + ): + with attempt: + new_keystone_passwd_on_mysql = juju_utils.leader_get( + self.application_name, KEYSTONE_PASSWD_KEY).strip() + new_keystone_passwd_conf = _get_password_from_keystone_leader() + self.assertNotEqual(old_keystone_passwd_on_mysql, + new_keystone_passwd_on_mysql) + self.assertNotEqual(old_keystone_passwd_conf, + new_keystone_passwd_conf) + self.assertEqual(new_keystone_passwd_on_mysql, + new_keystone_passwd_conf) + + # really wait for keystone to finish it's thing + zaza.model.block_until_all_units_idle() # finally, verify that keystone is still working. admin_keystone_session = ( From ce2b26a6e02322bb1a9433d77ee72f63e13b72ac Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 4 May 2023 09:31:40 +0200 Subject: [PATCH 5/7] Use json.loads() to read results of password rotation usernames --- zaza/openstack/charm_tests/mysql/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 2ed7f0f..c5e3f18 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -615,9 +615,9 @@ class MySQLInnoDBClusterRotatePasswordTests(MySQLCommonTests): 'list-service-usernames', action_params={} ) - usernames = action.data['results']['usernames'].split(',') - self.assertIn('keystone', usernames) + usernames = json.loads(action.data['results']['usernames']) logging.info("... usernames: %s", ', '.join(usernames)) + self.assertIn('keystone', usernames) # grab the password for keystone from the leader / to verify the change old_keystone_passwd_on_mysql = juju_utils.leader_get( From 0fe8e9d666dc79c99217ea86f74e7251474e6c52 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 5 May 2023 09:07:14 +0200 Subject: [PATCH 6/7] Switch from json.loads to yaml.safe_load for decoded usernames In the rotate password test, the results from the action return a list of strings that are deliminted by single quotes. This isn't compatible with json.loads(), but yaml.safe_load() is able to load the string as an array of strings. --- zaza/openstack/charm_tests/mysql/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index c5e3f18..830985b 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -21,6 +21,7 @@ import os import re import tempfile import tenacity +import yaml import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.model @@ -615,7 +616,7 @@ class MySQLInnoDBClusterRotatePasswordTests(MySQLCommonTests): 'list-service-usernames', action_params={} ) - usernames = json.loads(action.data['results']['usernames']) + usernames = yaml.safe_load(action.data['results']['usernames']) logging.info("... usernames: %s", ', '.join(usernames)) self.assertIn('keystone', usernames) From f0a6e802cced47ba3933c5fc6812f6ab1452d619 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 19 May 2023 12:36:28 +0100 Subject: [PATCH 7/7] Make username list on log look normal --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 830985b..ce9898d 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -617,7 +617,7 @@ class MySQLInnoDBClusterRotatePasswordTests(MySQLCommonTests): action_params={} ) usernames = yaml.safe_load(action.data['results']['usernames']) - logging.info("... usernames: %s", ', '.join(usernames)) + logging.info("... usernames: %s", usernames) self.assertIn('keystone', usernames) # grab the password for keystone from the leader / to verify the change