diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 70def89..8a1f959 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -1005,3 +1005,17 @@ class NeutronNetworkingVRRPTests(NeutronNetworkingBase): uc_neutron_client, gateway_hostname) self.check_connectivity(instance_1, instance_2) + + +class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super(NeutronOVSDeferredRestartTest, cls).setUpClass( + restart_config_file='/etc/neutron/neutron.conf', + test_service='neutron-openvswitch-agent', + restart_package='openvswitch-switch', + restart_package_service='openvswitch-switch', + application_name='neutron-openvswitch') diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index d4a3111..c3f03fb 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -428,3 +428,48 @@ class RmqTests(test_utils.OpenStackBaseTest): check_units(all_units) logging.info('OK') + + +class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass( + restart_config_file='/etc/rabbitmq/rabbitmq.config', + test_service='rabbitmq-server', + restart_package='rabbitmq-server', + restart_package_service='rabbitmq-server', + application_name='rabbitmq-server') + + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + for unit in zaza.model.get_units(self.application_name): + assert unit.workload_status_message in [ + 'Unit is ready', + 'Unit is ready and clustered'] + + def trigger_deferred_hook_via_charm(self): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be renderred in + applications config file. + + This overrides the base class version as the rabbit charm does not + have a debug option and the config file is not in oslo config format. + """ + app_config = zaza.model.get_application_config(self.application_name) + logging.info("Triggering deferred restart via config change") + new_debug_value = str( + int(app_config['connection-backlog'].get('value', 100) + 1)) + logging.info("Setting connection-backlog: {}".format(new_debug_value)) + zaza.model.set_application_config( + self.application_name, + {'connection-backlog': new_debug_value}) + logging.info("Waiting for units to be idle") + test_unit = zaza.model.get_units(self.application_name)[0] + zaza.model.block_until_unit_wl_message_match( + test_unit.entity_id, + status_pattern='.*config-changed.*') + zaza.model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 5a3e6c1..dec0cd0 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -18,6 +18,7 @@ import subprocess import sys import tenacity import unittest +import yaml import novaclient @@ -701,3 +702,292 @@ class OpenStackBaseTest(BaseCharmTest): instance_2 = self.retrieve_guest( '{}-ins-1'.format(self.RESOURCE_PREFIX)) return instance_1, instance_2 + + +class BaseDeferredRestartTest(OpenStackBaseTest): + """Check deferred restarts. + + Example of adding a deferred restart test:: + + class NeutronOVSDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + + @classmethod + def setUpClass(cls): + super(NeutronOVSDeferredRestartTest, cls).setUpClass( + restart_config_file = '/etc/neutron/neutron.conf', + test_service = 'neutron-openvswitch-agent', + restart_package = 'openvswitch-switch', + restart_package_service = 'openvswitch-switch', + application_name = 'neutron-openvswitch') + + NOTE: The test has been broken into various class methods which may require + specialisation if the charm being tested is not a standard OpenStack + charm e.g. `trigger_deferred_hook_via_charm` if the charm is not + an oslo config or does not have a debug option. + """ + + @classmethod + def setUpClass(cls, restart_config_file, test_service, restart_package, + restart_package_service, application_name): + """Run test setup. + + :param restart_config_file: Config file that will be changed to trigger + a service restart. + :type restart_config_file: str + :param test_service: Service that will require a restart after + restart_config_file has changed. + :type test_service: str + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + :param restart_package_service: Service that will require a restart + after restart_package has changed. + :type restart_package_service: str + :param application_name: Name of application to run tests against. + :type application_name: str + """ + cls.restart_config_file = restart_config_file + cls.test_service = test_service + cls.restart_package = restart_package + cls.restart_package_service = restart_package_service + cls.application_name = application_name + super(BaseDeferredRestartTest, cls).setUpClass( + application_name=cls.application_name) + + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + + def check_clear_restarts(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + model.run_action( + unit.entity_id, + 'restart-services', + action_params={'deferred': True}) + + # Check workload status no longer shows deferred restarts. + self.check_status_message_is_clear() + + def check_clear_hooks(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + model.run_action( + unit.entity_id, + 'run-deferred-hooks') + + # Check workload status no longer shows deferred restarts. + self.check_status_message_is_clear() + + def run_show_deferred_events_action(self): + """Run show-deferred-events and return results. + + :returns: Data from action run + :rtype: Dict + """ + unit = model.get_units(self.application_name)[0] + action = model.run_action(unit.entity_id, 'show-deferred-events') + return yaml.safe_load(action.data['results']['output']) + + def check_show_deferred_events_action_restart(self, test_service, + restart_reason): + """Check the output from the action to list deferred restarts. + + Run the action to list any deferred restarts and check it has entry for + the given service and reason. + + :param test_service: Service that should need a restart + :type test_service: str + :param restart_reason: The reason the action should list for the + service needing to be restarted. This can be a + substring. + :type restart_reason: str + """ + # Ensure that the deferred restart and cause are listed via action + logging.info( + ("Checking {} is marked as needing restart in " + "show-deferred-events action").format( + test_service)) + for event in self.run_show_deferred_events_action()['restarts']: + logging.info("{} in {} and {} in {}".format( + test_service, + event, + restart_reason, + event)) + if test_service in event and restart_reason in event: + break + else: + msg = 'No entry for restart of {} for reason {} found'.format( + test_service, + restart_reason) + raise Exception(msg) + + def check_show_deferred_events_action_hook(self, hook): + """Check the output from the action to list deferred eveents. + + Run the action to list any deferred events and check it has entry for + the given hook. + + :param hook: Hook name + :type hook: str + """ + # Ensure that the deferred restart and cause are listed via action + logging.info( + ("Checking {} is marked as skipped in " + "show-deferred-events action").format(hook)) + for event in self.run_show_deferred_events_action()['hooks']: + logging.info("{} in {}".format(hook, event)) + if hook in event: + break + else: + msg = '{} not found in {}'.format(hook, event) + raise Exception(msg) + + def check_show_deferred_restarts_wlm(self, test_service): + """Check the workload status message lists deferred restart. + + :param test_service: Service that should need a restart + :type test_service: str + """ + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + # Just checking one example service should we be checking all? + logging.info( + ("Checking {} is marked as needing restart in workload " + "message of {}".format(test_service, unit.entity_id))) + assert test_service in unit.workload_status_message + + def check_deferred_hook_wlm(self): + """Check the workload status message lists deferred event.""" + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + logging.info( + ("Checking {} is marked as having deferred hook in workload " + "message".format(unit.entity_id))) + assert 'config-changed' in unit.workload_status_message + + def trigger_deferred_hook_via_charm(self): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be renderred in + applications config file. + + NOTE: The implementation assumes the charm has a `debug` option and + self.restart_config_file in an oslo config file where that + debug option is renderred. If that is not true the specaliasation + class should override this method. + """ + app_config = model.get_application_config(self.application_name) + logging.info("Triggering deferred restart via config change") + new_debug_value = str(not app_config['debug']['value']) + logging.info("Setting debug: {}".format(new_debug_value)) + model.set_application_config( + self.application_name, + {'debug': new_debug_value}) + logging.info("Waiting for units to be idle") + test_unit = model.get_units(self.application_name)[0] + model.block_until_unit_wl_message_match( + test_unit.entity_id, + status_pattern='.*config-changed.*') + model.block_until_all_units_idle() + + def trigger_deferred_restart_via_package(self): + """Update a package which requires a service restart.""" + logging.info("Triggering deferred restart via package change") + # Test restart requested by package + for unit in model.get_units(self.application_name): + model.run_on_unit( + unit.entity_id, + 'dpkg-reconfigure {}; ./hooks/update-status'.format( + self.restart_package)) + + def run_charm_change_test(self): + """Trigger a deferred restart by updating a config file via the charm. + + Trigger a config-changed hook in the charm which will update a config + file and add a deferred restart. + + NOTE: If this test is not relevant for the target charm then override + this method and raise unittest.SkipTest + """ + self.trigger_deferred_hook_via_charm() + + self.check_deferred_hook_wlm() + self.check_show_deferred_events_action_hook('config-changed') + # Rerunning to flip config option back to previous value. + self.trigger_deferred_hook_via_charm() + logging.info("Running restart action to clear deferred restarts") + self.check_clear_hooks() + + def run_package_change_test(self): + """Trigger a deferred restart by updating a package. + + Update a package which requires will add a deferred restart. + + NOTE: If this test is not relevant for the target charm then override + this method and raise unittest.SkipTest + """ + self.trigger_deferred_restart_via_package() + + self.check_show_deferred_restarts_wlm(self.restart_package_service) + self.check_show_deferred_events_action_restart( + self.restart_package_service, + 'Package update') + logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def test_deferred_restarts(self): + """Run deferred restart tests.""" + app_config = model.get_application_config(self.application_name) + auto_restart_config_key = 'enable-auto-restarts' + if auto_restart_config_key not in app_config: + raise unittest.SkipTest("Deferred restarts not implemented") + + # Ensure auto restarts are off. + policy_file = '/etc/policy-rc.d/charm-{}.policy'.format( + self.application_name) + if app_config[auto_restart_config_key]['value']: + logging.info("Turning off auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'False'}) + logging.info("Waiting for {} to appear on units of {}".format( + policy_file, + self.application_name)) + model.block_until_file_has_contents( + self.application_name, + policy_file, + 'policy_requestor_name') + # The block_until_file_has_contents ensures the change we waiting + # for has happened, now just wait for any hooks to finish. + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + else: + logging.info("Auto restarts already disabled") + + # Trigger a config change which requires a restart + self.run_charm_change_test() + + # Trigger a package change which requires a restart + self.run_package_change_test() + + # Finished so turn auto-restarts back on. + logging.info("Turning on auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'True'}) + model.block_until_file_missing( + self.application_name, + policy_file) + model.block_until_all_units_idle()