From b29c11720e7504d72535c8598baaa33071445c27 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 23 Mar 2023 15:11:58 +0000 Subject: [PATCH 1/5] Add test to rotate rabbitmq-server service user password This new test verifies that cinder can have its password rotated and then still operate afterwards. It verifies that the on-disk password is changed in the cinder application and that the user list can be performed. --- .../charm_tests/rabbitmq_server/tests.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 9b18d8d..b181519 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -14,8 +14,10 @@ """RabbitMQ Testing.""" +import configparser import json import logging +import re import time import uuid import unittest @@ -25,6 +27,8 @@ import tenacity import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.juju as juju_utils +import zaza.openstack.utilities.openstack as openstack_utils from zaza.openstack.utilities.generic import get_series from zaza.openstack.utilities.os_versions import CompareHostReleases @@ -470,6 +474,93 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK') +class RmqRotateServiceUserPasswordTests(test_utils.OpenStackBaseTest): + """RMQ service user password rotation tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running RMQ service user rotation tests.""" + super().setUpClass() + cls.application = "rabbitmq-server" + + def test_rotate_cinder_service_user_password(self): + """Verify action used to rotate a service user (cinder) password.""" + CINDER_APP = 'cinder' + CINDER_PASSWD_KEY = "cinder.passwd" + CINDER_CONF_FILE = '/etc/cinder/cinder.conf' + + def _get_password_from_cinder_leader(): + conf = zaza.model.file_contents( + 'cinder/leader', CINDER_CONF_FILE) + config = configparser.ConfigParser() + config.read_string(conf) + connection_info = config['DEFAULT']['transport'] + match = re.match(r"^rabbit.*cinder:(.+)@", connection_info) + if match: + return match[1] + self.fail("Couldn't find mysql password in {}" + .format(connection_info)) + + # only do the test if keystone is in the model + applications = zaza.model.sync_deployed(self.model_name) + if CINDER_APP not in applications: + self.skipTest( + '{} is not deployed, so not doing password rotation' + .format(CINDER_APP)) + + # get the users via the 'list-service-usernames' action. + logging.info( + "Getting usernames from rabbitmq-server that can have password " + "rotated.") + action = zaza.model.run_action_on_leader( + self.application, + 'list-service-usernames', + action_params={} + ) + usernames = action.data['results']['usernames'] + self.assertIn(CINDER_APP, usernames) + logging.info("... usernames: %s", ', '.join(usernames)) + + # grab the password for cinder from the leader / to verify the change + old_cinder_passwd_on_rmq = juju_utils.leader_get( + self.application_name, CINDER_PASSWD_KEY).strip() + old_cinder_passwd_conf = _get_password_from_cinder_leader() + + # verify that cinder is working. + cinder_client = openstack_utils.get_cinder_session_client( + self.keystone_session, version=3.42) + cinder_client.volumes.list() + + # now rotate the password for keystone + # run the action to rotate the password. + logging.info("Rotating password for cinder in rabbitmq-server.") + zaza.model.run_action_on_leader( + self.application_name, + 'rotate-service-user-password', + action_params={'service-user': 'cinder'}, + ) + + # let everything settle. + logging.info("Waiting for model to settle.") + zaza.model.block_until_all_units_idle() + + # verify that the password has changed. + new_cinder_passwd_on_rmq = juju_utils.leader_get( + self.application_name, CINDER_PASSWD_KEY).strip() + new_cinder_passwd_conf = _get_password_from_cinder_leader() + self.assertNotEqual(old_cinder_passwd_on_rmq, + new_cinder_passwd_on_rmq) + self.assertNotEqual(old_cinder_passwd_conf, + new_cinder_passwd_conf) + self.assertEqual(new_cinder_passwd_on_rmq, + new_cinder_passwd_conf) + + # finally, verify that cinder is still working. + cinder_client = openstack_utils.get_cinder_session_client( + self.keystone_session, version=3.42) + cinder_client.volumes.list() + + class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): """Deferred restart tests.""" From d964696f48763bc4e2b287163e24508c79a81fc0 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 24 Mar 2023 12:36:25 +0000 Subject: [PATCH 2/5] Add additional wait_for_agent_status() check This is to ensure that the model really is settled after the password rotate action. --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index b181519..91aa041 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -542,6 +542,7 @@ class RmqRotateServiceUserPasswordTests(test_utils.OpenStackBaseTest): # let everything settle. logging.info("Waiting for model to settle.") + zaza.model.wait_for_agent_status() zaza.model.block_until_all_units_idle() # verify that the password has changed. From 17c0de8810a1b05096560cf113b241201b0b587f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 29 Mar 2023 12:26:22 +0100 Subject: [PATCH 3/5] Fix handling of usernames in rabbitmq password rotation test The usernames were not handled correctly. The return value from looking up the action value is a string, and so it needs to be split on ','. --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 91aa041..86ec92f 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -517,7 +517,7 @@ class RmqRotateServiceUserPasswordTests(test_utils.OpenStackBaseTest): 'list-service-usernames', action_params={} ) - usernames = action.data['results']['usernames'] + usernames = action.data['results']['usernames'].split(',') self.assertIn(CINDER_APP, usernames) logging.info("... usernames: %s", ', '.join(usernames)) From 6bfbab25785e7b712386cc07f6b4cc7c426e2cba Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 4 May 2023 09:28:57 +0200 Subject: [PATCH 4/5] Use json.loads() for password rotation usernames result --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 86ec92f..7a84d4a 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -517,7 +517,7 @@ class RmqRotateServiceUserPasswordTests(test_utils.OpenStackBaseTest): 'list-service-usernames', action_params={} ) - usernames = action.data['results']['usernames'].split(',') + usernames = json.loads(action.data['results']['usernames']) self.assertIn(CINDER_APP, usernames) logging.info("... usernames: %s", ', '.join(usernames)) From 6516a340453f525ea565b95e120c3e13979e63ab Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 5 May 2023 08:59:13 +0200 Subject: [PATCH 5/5] Update rabbitmq rotate password test to be more robust/correct Fix a bug where the wrong paramter from the /etc/cinder/cinder.conf file is being used ('transport' -> 'transport_url'). Fix issue around using json instead of yaml for loading the usernames from the action data from the list_service_usernames action, which uses single quotes to quote the data, which isn't compatible with json. Make the testing for the password change in the cinder.conf more robust by retrying, as juju is essentially async and it's difficult to determine exactly when the hook will get fired on cinder. --- .../charm_tests/rabbitmq_server/tests.py | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 7a84d4a..0dc1a0e 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -21,6 +21,7 @@ import re import time import uuid import unittest +import yaml import juju import tenacity @@ -494,7 +495,7 @@ class RmqRotateServiceUserPasswordTests(test_utils.OpenStackBaseTest): 'cinder/leader', CINDER_CONF_FILE) config = configparser.ConfigParser() config.read_string(conf) - connection_info = config['DEFAULT']['transport'] + connection_info = config['DEFAULT']['transport_url'] match = re.match(r"^rabbit.*cinder:(.+)@", connection_info) if match: return match[1] @@ -517,7 +518,7 @@ class RmqRotateServiceUserPasswordTests(test_utils.OpenStackBaseTest): 'list-service-usernames', action_params={} ) - usernames = json.loads(action.data['results']['usernames']) + usernames = yaml.safe_load(action.data['results']['usernames']) self.assertIn(CINDER_APP, usernames) logging.info("... usernames: %s", ', '.join(usernames)) @@ -546,15 +547,25 @@ class RmqRotateServiceUserPasswordTests(test_utils.OpenStackBaseTest): zaza.model.block_until_all_units_idle() # verify that the password has changed. - new_cinder_passwd_on_rmq = juju_utils.leader_get( - self.application_name, CINDER_PASSWD_KEY).strip() - new_cinder_passwd_conf = _get_password_from_cinder_leader() - self.assertNotEqual(old_cinder_passwd_on_rmq, - new_cinder_passwd_on_rmq) - self.assertNotEqual(old_cinder_passwd_conf, - new_cinder_passwd_conf) - self.assertEqual(new_cinder_passwd_on_rmq, - new_cinder_passwd_conf) + # Due to the async-ness of the whole model and when the various hooks + # will fire between rabbitmq-server and cinder, we retry a reasonable + # time to wait for everything to propagate through to the + # /etc/cinder/cinder.conf file. + for attempt in tenacity.Retrying( + reraise=True, + wait=tenacity.wait_fixed(30), + stop=tenacity.stop_after_attempt(20), # wait for max 10m + ): + with attempt: + new_cinder_passwd_on_rmq = juju_utils.leader_get( + self.application_name, CINDER_PASSWD_KEY).strip() + new_cinder_passwd_conf = _get_password_from_cinder_leader() + self.assertNotEqual(old_cinder_passwd_on_rmq, + new_cinder_passwd_on_rmq) + self.assertNotEqual(old_cinder_passwd_conf, + new_cinder_passwd_conf) + self.assertEqual(new_cinder_passwd_on_rmq, + new_cinder_passwd_conf) # finally, verify that cinder is still working. cinder_client = openstack_utils.get_cinder_session_client(