From 2054b7c4775ea8c79d99d60f23a684f6e1910db3 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 25 Feb 2021 09:16:34 +0000 Subject: [PATCH 1/7] Add test for deferred restarts --- zaza/openstack/charm_tests/neutron/tests.py | 14 ++ zaza/openstack/charm_tests/test_utils.py | 236 ++++++++++++++++++++ 2 files changed, 250 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index bebe3b6..412a352 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -952,3 +952,17 @@ class NeutronNetworkingVRRPTests(NeutronNetworkingBase): uc_neutron_client, gateway_hostname) self.check_connectivity(instance_1, instance_2) + + +class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super(NeutronOVSDeferredRestartTest, cls).setUpClass( + restart_config_file='/etc/neutron/neutron.conf', + test_service='neutron-openvswitch-agent', + restart_package='openvswitch-switch', + restart_package_service='openvswitch-switch', + application_name='neutron-openvswitch') diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 5a3e6c1..bd82a38 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -18,6 +18,7 @@ import subprocess import sys import tenacity import unittest +import yaml import novaclient @@ -701,3 +702,238 @@ class OpenStackBaseTest(BaseCharmTest): instance_2 = self.retrieve_guest( '{}-ins-1'.format(self.RESOURCE_PREFIX)) return instance_1, instance_2 + + +class BaseDeferredRestartTest(OpenStackBaseTest): + """Check deferred restarts. + + Example of adding a deferred restart test:: + + class NeutronOVSDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + + @classmethod + def setUpClass(cls): + super(NeutronOVSDeferredRestartTest, cls).setUpClass( + restart_config_file = '/etc/neutron/neutron.conf', + test_service = 'neutron-openvswitch-agent', + restart_package = 'openvswitch-switch', + restart_package_service = 'openvswitch-switch', + application_name = 'neutron-openvswitch') + + NOTE: The test has been broken into various class methods which may require + specialisation if the charm being tested is not a standard OpenStack + charm e.g. `trigger_deferred_restart_via_charm` if the charm is not + an oslo config or does not have a debug option. + """ + + @classmethod + def setUpClass(cls, restart_config_file, test_service, restart_package, + restart_package_service, application_name): + """Run test setup. + + :param restart_config_file: Config file that will be changed to trigger + a service restart. + :type restart_config_file: str + :param test_service: Service that will require a restart after + restart_config_file has changed. + :type test_service: str + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + :param restart_package_service: Service that will require a restart + after restart_package has changed. + :type restart_package_service: str + :param application_name: Name of application to run tests against. + :type application_name: str + """ + cls.restart_config_file = restart_config_file + cls.test_service = test_service + cls.restart_package = restart_package + cls.restart_package_service = restart_package_service + cls.application_name = application_name + super(BaseDeferredRestartTest, cls).setUpClass( + application_name=cls.application_name) + + def check_clear_restarts(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + model.run_action( + unit.entity_id, + 'restart-services', + action_params={'deferred': True}) + + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + + def check_show_deferred_restarts_action(self, test_service, + restart_reason): + """Check the output from the action to list deferred restarts. + + Run the action to list any deferred restarts and check it has entry for + the given service and reason. + + :param test_service: Service that should need a restart + :type test_service: str + :param restart_reason: The reason the action should list for the + service needing to be restarted. This can be a + substring. + :type restart_reason: str + """ + # Ensure that the deferred restart and cause are listed via action + for unit in model.get_units(self.application_name): + action = model.run_action( + unit.entity_id, + 'show-deferred-restarts') + logging.info( + ("Checking {} is marked as needing restart in " + "show-deferred-restarts action on {}").format( + test_service, + unit.entity_id)) + for event in yaml.safe_load(action.data['results']['output']): + if test_service in event and restart_reason in event: + break + else: + msg = 'No entry for restart of {} for reason {} found'.format( + test_service, + restart_reason) + raise Exception(msg) + + def check_show_deferred_restarts_wlm(self, test_service): + """Check the workload status message lists deferred restart. + + :param test_service: Service that should need a restart + :type test_service: str + """ + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + # Just checking one example service should we be checking all? + logging.info( + ("Checking {} is marked as needing restart in workload " + "message of {}".format(test_service, unit.entity_id))) + assert test_service in unit.workload_status_message + + def trigger_deferred_restart_via_charm(self): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be renderred in + applications config file. + + NOTE: The implementation assumes the charm has a `debug` option and + self.restart_config_file in an oslo config file where that + debug option is renderred. If that is not true the specaliasation + class should override this method. + """ + app_config = model.get_application_config(self.application_name) + logging.info("Triggering deferred restart via config change") + new_debug_value = str(not app_config['debug']['value']) + logging.info("Setting debug: {}".format(new_debug_value)) + model.set_application_config( + self.application_name, + {'debug': new_debug_value}) + expected_contents = { + 'DEFAULT': { + 'debug': [new_debug_value]}} + logging.info("Waiting for debug to be {} in {}".format( + new_debug_value, + self.restart_config_file)) + model.block_until_oslo_config_entries_match( + self.application_name, + self.restart_config_file, + expected_contents) + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + + def trigger_deferred_restart_via_package(self): + """Update a package which requires a service restart.""" + logging.info("Triggering deferred restart via package change") + # Test restart requested by package + for unit in model.get_units(self.application_name): + model.run_on_unit( + unit.entity_id, + 'dpkg-reconfigure {}; ./hooks/update-status'.format( + self.restart_package)) + + def run_charm_change_test(self): + """Trigger a deferred restart by updating a config file via the charm. + + Trigger a config-changed hook in the charm which will update a config + file and add a deferred restart. + + NOTE: If this test is not relevant for the target charm then override + this method and raise unittest.SkipTest + """ + self.trigger_deferred_restart_via_charm() + + self.check_show_deferred_restarts_wlm(self.test_service) + self.check_show_deferred_restarts_action( + self.test_service, + self.restart_config_file) + logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def run_package_change_test(self): + """Trigger a deferred restart by updating a package. + + Update a package which requires will add a deferred restart. + + NOTE: If this test is not relevant for the target charm then override + this method and raise unittest.SkipTest + """ + self.trigger_deferred_restart_via_package() + + self.check_show_deferred_restarts_wlm(self.restart_package_service) + self.check_show_deferred_restarts_action( + self.restart_package_service, + 'Pkg Update') + logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def test_deferred_restarts(self): + """Run deferred restart tests.""" + app_config = model.get_application_config(self.application_name) + auto_restart_config_key = 'enable-auto-restarts' + if auto_restart_config_key not in app_config: + raise unittest.SkipTest("Deferred restarts not implemented") + + # Ensure auto restarts are off. + policy_file = '/etc/policy-rc.d/charm-{}.policy'.format( + self.application_name) + if app_config[auto_restart_config_key]['value']: + logging.info("Turning off auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'False'}) + logging.info("Waiting for {} to appear on units of {}".format( + policy_file, + self.application_name)) + model.block_until_file_has_contents( + self.application_name, + policy_file, + 'policy_requestor_name') + # The block_until_file_has_contents ensures the change we waiting + # for has happened, now just wait for any hooks to finish. + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + else: + logging.info("Auto restarts already disabled") + + # Trigger a config change which requires a restart + self.run_charm_change_test() + + # Trigger a package change which requires a restart + self.run_package_change_test() + + # Finished so turn auto-restarts back on. + logging.info("Turning on auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'True'}) + model.block_until_file_missing( + self.application_name, + policy_file) + model.block_until_all_units_idle() From d4b8fbaed037df0fb9050c9837c5f9efdc33455d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 25 Feb 2021 11:01:22 +0000 Subject: [PATCH 2/7] Rerun self.trigger_deferred_restart_via_charm --- zaza/openstack/charm_tests/test_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index bd82a38..3aff816 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -875,6 +875,8 @@ class BaseDeferredRestartTest(OpenStackBaseTest): self.check_show_deferred_restarts_action( self.test_service, self.restart_config_file) + # Rerunning to flip config option back to previous value. + self.trigger_deferred_restart_via_charm() logging.info("Running restart action to clear deferred restarts") self.check_clear_restarts() From 1f47ef0bd769e3e7976130d9c81071d84bacd60f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 22 Mar 2021 18:43:13 +0000 Subject: [PATCH 3/7] Update package restart message afer ch change --- zaza/openstack/charm_tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 3aff816..ea134d3 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -893,7 +893,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): self.check_show_deferred_restarts_wlm(self.restart_package_service) self.check_show_deferred_restarts_action( self.restart_package_service, - 'Pkg Update') + 'Package update') logging.info("Running restart action to clear deferred restarts") self.check_clear_restarts() From 085d5a1513afbf17828ae2bbf3e2c60bd8f0eb02 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 23 Mar 2021 15:57:21 +0000 Subject: [PATCH 4/7] Add RabbitMQDeferredRestartTest + fixes --- .../charm_tests/rabbitmq_server/tests.py | 42 +++++++++++++++++++ zaza/openstack/charm_tests/test_utils.py | 3 +- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index d4a3111..40e227c 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -428,3 +428,45 @@ class RmqTests(test_utils.OpenStackBaseTest): check_units(all_units) logging.info('OK') + + +class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass( + restart_config_file='/etc/rabbitmq/rabbitmq.config', + test_service='rabbitmq-server', + restart_package='rabbitmq-server', + restart_package_service='rabbitmq-server', + application_name='rabbitmq-server') + + def trigger_deferred_restart_via_charm(self): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be renderred in + applications config file. + + NOTE: The implementation assumes the charm has a `debug` option and + self.restart_config_file in an oslo config file where that + debug option is renderred. If that is not true the specaliasation + class should override this method. + """ + app_config = zaza.model.get_application_config(self.application_name) + logging.info("Triggering deferred restart via config change") + new_debug_value = str(int(app_config['connection-backlog'].get('value', 100) + 1)) + logging.info("Setting connection-backlog: {}".format(new_debug_value)) + zaza.model.set_application_config( + self.application_name, + {'connection-backlog': new_debug_value}) + logging.info("Waiting for connection-backlog to be {} in {}".format( + new_debug_value, + self.restart_config_file)) + zaza.model.block_until_file_matches_re( + self.application_name, + self.restart_config_file, + '{{backlog, {}}}'.format(new_debug_value)) + logging.info("Waiting for units to be idle") + zaza.model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index ea134d3..26a9917 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -770,7 +770,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): # Check workload status no longer shows deferred restarts. for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + assert 'Services queued' not in unit.workload_status_message def check_show_deferred_restarts_action(self, test_service, restart_reason): @@ -796,6 +796,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): "show-deferred-restarts action on {}").format( test_service, unit.entity_id)) + assert action.data['status'] == 'completed' for event in yaml.safe_load(action.data['results']['output']): if test_service in event and restart_reason in event: break From 77cb2d0bd84f1a095ac034be6fa428a8242fe0d8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 24 Mar 2021 13:52:34 +0000 Subject: [PATCH 5/7] Fix doc string and lint --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 40e227c..14fadc4 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -449,14 +449,13 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): Set the charm debug option and wait for that change to be renderred in applications config file. - NOTE: The implementation assumes the charm has a `debug` option and - self.restart_config_file in an oslo config file where that - debug option is renderred. If that is not true the specaliasation - class should override this method. + This overrides the base class version as the rabbit charm does not + have a debug option and the config file is not in oslo config format. """ app_config = zaza.model.get_application_config(self.application_name) logging.info("Triggering deferred restart via config change") - new_debug_value = str(int(app_config['connection-backlog'].get('value', 100) + 1)) + new_debug_value = str( + int(app_config['connection-backlog'].get('value', 100) + 1)) logging.info("Setting connection-backlog: {}".format(new_debug_value)) zaza.model.set_application_config( self.application_name, From 0fdd876213079cfd1adfdea2d29ee23eb87bc2a0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 29 Mar 2021 10:41:46 +0000 Subject: [PATCH 6/7] Update inline with recent charm changes --- .../charm_tests/rabbitmq_server/tests.py | 13 +- zaza/openstack/charm_tests/test_utils.py | 129 ++++++++++++------ 2 files changed, 93 insertions(+), 49 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 14fadc4..13d0e52 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -443,7 +443,7 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): restart_package_service='rabbitmq-server', application_name='rabbitmq-server') - def trigger_deferred_restart_via_charm(self): + def trigger_deferred_hook_via_charm(self): """Set charm config option which requires a service start. Set the charm debug option and wait for that change to be renderred in @@ -460,12 +460,9 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): zaza.model.set_application_config( self.application_name, {'connection-backlog': new_debug_value}) - logging.info("Waiting for connection-backlog to be {} in {}".format( - new_debug_value, - self.restart_config_file)) - zaza.model.block_until_file_matches_re( - self.application_name, - self.restart_config_file, - '{{backlog, {}}}'.format(new_debug_value)) logging.info("Waiting for units to be idle") + test_unit = zaza.model.get_units(self.application_name)[0] + zaza.model.block_until_unit_wl_message_match( + test_unit.entity_id, + status_pattern='.*config-changed.*') zaza.model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 26a9917..94ac235 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -723,7 +723,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): NOTE: The test has been broken into various class methods which may require specialisation if the charm being tested is not a standard OpenStack - charm e.g. `trigger_deferred_restart_via_charm` if the charm is not + charm e.g. `trigger_deferred_hook_via_charm` if the charm is not an oslo config or does not have a debug option. """ @@ -770,10 +770,36 @@ class BaseDeferredRestartTest(OpenStackBaseTest): # Check workload status no longer shows deferred restarts. for unit in model.get_units(self.application_name): - assert 'Services queued' not in unit.workload_status_message + assert unit.workload_status_message == 'Unit is ready' - def check_show_deferred_restarts_action(self, test_service, - restart_reason): + def check_clear_hooks(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + model.run_action( + unit.entity_id, + 'run-deferred-hooks') + + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + + def run_show_deferred_events_action(self): + """Run show-deferred-events and return results. + + :returns: Data from action run + :rtype: Dict + """ + unit = model.get_units(self.application_name)[0] + action = model.run_action(unit.entity_id, 'show-deferred-events') + return yaml.safe_load(action.data['results']['output']) + + def check_show_deferred_events_action_restart(self, test_service, + restart_reason): """Check the output from the action to list deferred restarts. Run the action to list any deferred restarts and check it has entry for @@ -787,24 +813,44 @@ class BaseDeferredRestartTest(OpenStackBaseTest): :type restart_reason: str """ # Ensure that the deferred restart and cause are listed via action - for unit in model.get_units(self.application_name): - action = model.run_action( - unit.entity_id, - 'show-deferred-restarts') - logging.info( - ("Checking {} is marked as needing restart in " - "show-deferred-restarts action on {}").format( - test_service, - unit.entity_id)) - assert action.data['status'] == 'completed' - for event in yaml.safe_load(action.data['results']['output']): - if test_service in event and restart_reason in event: - break - else: - msg = 'No entry for restart of {} for reason {} found'.format( - test_service, - restart_reason) - raise Exception(msg) + logging.info( + ("Checking {} is marked as needing restart in " + "show-deferred-events action").format( + test_service)) + for event in self.run_show_deferred_events_action()['restarts']: + logging.info("{} in {} and {} in {}".format( + test_service, + event, + restart_reason, + event)) + if test_service in event and restart_reason in event: + break + else: + msg = 'No entry for restart of {} for reason {} found'.format( + test_service, + restart_reason) + raise Exception(msg) + + def check_show_deferred_events_action_hook(self, hook): + """Check the output from the action to list deferred eveents. + + Run the action to list any deferred events and check it has entry for + the given hook. + + :param hook: Hook name + :type hook: str + """ + # Ensure that the deferred restart and cause are listed via action + logging.info( + ("Checking {} is marked as skipped in " + "show-deferred-events action").format(hook)) + for event in self.run_show_deferred_events_action()['hooks']: + logging.info("{} in {}".format(hook, event)) + if hook in event: + break + else: + msg = '{} not found in {}'.format(hook, event) + raise Exception(msg) def check_show_deferred_restarts_wlm(self, test_service): """Check the workload status message lists deferred restart. @@ -820,7 +866,16 @@ class BaseDeferredRestartTest(OpenStackBaseTest): "message of {}".format(test_service, unit.entity_id))) assert test_service in unit.workload_status_message - def trigger_deferred_restart_via_charm(self): + def check_deferred_hook_wlm(self): + """Check the workload status message lists deferred event.""" + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + logging.info( + ("Checking {} is marked as having deferred hook in workload " + "message".format(unit.entity_id))) + assert 'config-changed' in unit.workload_status_message + + def trigger_deferred_hook_via_charm(self): """Set charm config option which requires a service start. Set the charm debug option and wait for that change to be renderred in @@ -838,17 +893,11 @@ class BaseDeferredRestartTest(OpenStackBaseTest): model.set_application_config( self.application_name, {'debug': new_debug_value}) - expected_contents = { - 'DEFAULT': { - 'debug': [new_debug_value]}} - logging.info("Waiting for debug to be {} in {}".format( - new_debug_value, - self.restart_config_file)) - model.block_until_oslo_config_entries_match( - self.application_name, - self.restart_config_file, - expected_contents) logging.info("Waiting for units to be idle") + test_unit = model.get_units(self.application_name)[0] + model.block_until_unit_wl_message_match( + test_unit.entity_id, + status_pattern='.*config-changed.*') model.block_until_all_units_idle() def trigger_deferred_restart_via_package(self): @@ -870,16 +919,14 @@ class BaseDeferredRestartTest(OpenStackBaseTest): NOTE: If this test is not relevant for the target charm then override this method and raise unittest.SkipTest """ - self.trigger_deferred_restart_via_charm() + self.trigger_deferred_hook_via_charm() - self.check_show_deferred_restarts_wlm(self.test_service) - self.check_show_deferred_restarts_action( - self.test_service, - self.restart_config_file) + self.check_deferred_hook_wlm() + self.check_show_deferred_events_action_hook('config-changed') # Rerunning to flip config option back to previous value. - self.trigger_deferred_restart_via_charm() + self.trigger_deferred_hook_via_charm() logging.info("Running restart action to clear deferred restarts") - self.check_clear_restarts() + self.check_clear_hooks() def run_package_change_test(self): """Trigger a deferred restart by updating a package. @@ -892,7 +939,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): self.trigger_deferred_restart_via_package() self.check_show_deferred_restarts_wlm(self.restart_package_service) - self.check_show_deferred_restarts_action( + self.check_show_deferred_events_action_restart( self.restart_package_service, 'Package update') logging.info("Running restart action to clear deferred restarts") From f78105a6460f9338d756bab160ddf13b295627a0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 29 Mar 2021 14:46:04 +0000 Subject: [PATCH 7/7] Allow status message to be overridden --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 7 +++++++ zaza/openstack/charm_tests/test_utils.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 13d0e52..c3f03fb 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -443,6 +443,13 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): restart_package_service='rabbitmq-server', application_name='rabbitmq-server') + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + for unit in zaza.model.get_units(self.application_name): + assert unit.workload_status_message in [ + 'Unit is ready', + 'Unit is ready and clustered'] + def trigger_deferred_hook_via_charm(self): """Set charm config option which requires a service start. diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 94ac235..dec0cd0 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -755,6 +755,12 @@ class BaseDeferredRestartTest(OpenStackBaseTest): super(BaseDeferredRestartTest, cls).setUpClass( application_name=cls.application_name) + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + def check_clear_restarts(self): """Clear and deferred restarts and check status. @@ -769,8 +775,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): action_params={'deferred': True}) # Check workload status no longer shows deferred restarts. - for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + self.check_status_message_is_clear() def check_clear_hooks(self): """Clear and deferred restarts and check status. @@ -785,8 +790,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): 'run-deferred-hooks') # Check workload status no longer shows deferred restarts. - for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + self.check_status_message_is_clear() def run_show_deferred_events_action(self): """Run show-deferred-events and return results.