Merge pull request #508 from gnuoy/deferred-restart-tests
Add test for deferred restarts
This commit is contained in:
David Ames
GitHub
@@ -1005,3 +1005,17 @@ class NeutronNetworkingVRRPTests(NeutronNetworkingBase):
|
||||
uc_neutron_client,
|
||||
gateway_hostname)
|
||||
self.check_connectivity(instance_1, instance_2)
|
||||
|
||||
|
||||
class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest):
|
||||
"""Deferred restart tests."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for deferred restart tests."""
|
||||
super(NeutronOVSDeferredRestartTest, cls).setUpClass(
|
||||
restart_config_file='/etc/neutron/neutron.conf',
|
||||
test_service='neutron-openvswitch-agent',
|
||||
restart_package='openvswitch-switch',
|
||||
restart_package_service='openvswitch-switch',
|
||||
application_name='neutron-openvswitch')
|
||||
|
||||
@@ -428,3 +428,48 @@ class RmqTests(test_utils.OpenStackBaseTest):
|
||||
check_units(all_units)
|
||||
|
||||
logging.info('OK')
|
||||
|
||||
|
||||
class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest):
|
||||
"""Deferred restart tests."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for deferred restart tests."""
|
||||
super().setUpClass(
|
||||
restart_config_file='/etc/rabbitmq/rabbitmq.config',
|
||||
test_service='rabbitmq-server',
|
||||
restart_package='rabbitmq-server',
|
||||
restart_package_service='rabbitmq-server',
|
||||
application_name='rabbitmq-server')
|
||||
|
||||
def check_status_message_is_clear(self):
|
||||
"""Check each units status message show no defeerred events."""
|
||||
for unit in zaza.model.get_units(self.application_name):
|
||||
assert unit.workload_status_message in [
|
||||
'Unit is ready',
|
||||
'Unit is ready and clustered']
|
||||
|
||||
def trigger_deferred_hook_via_charm(self):
|
||||
"""Set charm config option which requires a service start.
|
||||
|
||||
Set the charm debug option and wait for that change to be renderred in
|
||||
applications config file.
|
||||
|
||||
This overrides the base class version as the rabbit charm does not
|
||||
have a debug option and the config file is not in oslo config format.
|
||||
"""
|
||||
app_config = zaza.model.get_application_config(self.application_name)
|
||||
logging.info("Triggering deferred restart via config change")
|
||||
new_debug_value = str(
|
||||
int(app_config['connection-backlog'].get('value', 100) + 1))
|
||||
logging.info("Setting connection-backlog: {}".format(new_debug_value))
|
||||
zaza.model.set_application_config(
|
||||
self.application_name,
|
||||
{'connection-backlog': new_debug_value})
|
||||
logging.info("Waiting for units to be idle")
|
||||
test_unit = zaza.model.get_units(self.application_name)[0]
|
||||
zaza.model.block_until_unit_wl_message_match(
|
||||
test_unit.entity_id,
|
||||
status_pattern='.*config-changed.*')
|
||||
zaza.model.block_until_all_units_idle()
|
||||
|
||||
@@ -18,6 +18,7 @@ import subprocess
|
||||
import sys
|
||||
import tenacity
|
||||
import unittest
|
||||
import yaml
|
||||
|
||||
import novaclient
|
||||
|
||||
@@ -701,3 +702,292 @@ class OpenStackBaseTest(BaseCharmTest):
|
||||
instance_2 = self.retrieve_guest(
|
||||
'{}-ins-1'.format(self.RESOURCE_PREFIX))
|
||||
return instance_1, instance_2
|
||||
|
||||
|
||||
class BaseDeferredRestartTest(OpenStackBaseTest):
|
||||
"""Check deferred restarts.
|
||||
|
||||
Example of adding a deferred restart test::
|
||||
|
||||
class NeutronOVSDeferredRestartTest(
|
||||
test_utils.BaseDeferredRestartTest):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super(NeutronOVSDeferredRestartTest, cls).setUpClass(
|
||||
restart_config_file = '/etc/neutron/neutron.conf',
|
||||
test_service = 'neutron-openvswitch-agent',
|
||||
restart_package = 'openvswitch-switch',
|
||||
restart_package_service = 'openvswitch-switch',
|
||||
application_name = 'neutron-openvswitch')
|
||||
|
||||
NOTE: The test has been broken into various class methods which may require
|
||||
specialisation if the charm being tested is not a standard OpenStack
|
||||
charm e.g. `trigger_deferred_hook_via_charm` if the charm is not
|
||||
an oslo config or does not have a debug option.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls, restart_config_file, test_service, restart_package,
|
||||
restart_package_service, application_name):
|
||||
"""Run test setup.
|
||||
|
||||
:param restart_config_file: Config file that will be changed to trigger
|
||||
a service restart.
|
||||
:type restart_config_file: str
|
||||
:param test_service: Service that will require a restart after
|
||||
restart_config_file has changed.
|
||||
:type test_service: str
|
||||
:param restart_package: Package that will be changed to trigger a
|
||||
service restart.
|
||||
:type restart_package: str
|
||||
:param restart_package_service: Service that will require a restart
|
||||
after restart_package has changed.
|
||||
:type restart_package_service: str
|
||||
:param application_name: Name of application to run tests against.
|
||||
:type application_name: str
|
||||
"""
|
||||
cls.restart_config_file = restart_config_file
|
||||
cls.test_service = test_service
|
||||
cls.restart_package = restart_package
|
||||
cls.restart_package_service = restart_package_service
|
||||
cls.application_name = application_name
|
||||
super(BaseDeferredRestartTest, cls).setUpClass(
|
||||
application_name=cls.application_name)
|
||||
|
||||
def check_status_message_is_clear(self):
|
||||
"""Check each units status message show no defeerred events."""
|
||||
# Check workload status no longer shows deferred restarts.
|
||||
for unit in model.get_units(self.application_name):
|
||||
assert unit.workload_status_message == 'Unit is ready'
|
||||
|
||||
def check_clear_restarts(self):
|
||||
"""Clear and deferred restarts and check status.
|
||||
|
||||
Clear and deferred restarts and then check the workload status message
|
||||
for each unit.
|
||||
"""
|
||||
# Use action to run any deferred restarts
|
||||
for unit in model.get_units(self.application_name):
|
||||
model.run_action(
|
||||
unit.entity_id,
|
||||
'restart-services',
|
||||
action_params={'deferred': True})
|
||||
|
||||
# Check workload status no longer shows deferred restarts.
|
||||
self.check_status_message_is_clear()
|
||||
|
||||
def check_clear_hooks(self):
|
||||
"""Clear and deferred restarts and check status.
|
||||
|
||||
Clear and deferred restarts and then check the workload status message
|
||||
for each unit.
|
||||
"""
|
||||
# Use action to run any deferred restarts
|
||||
for unit in model.get_units(self.application_name):
|
||||
model.run_action(
|
||||
unit.entity_id,
|
||||
'run-deferred-hooks')
|
||||
|
||||
# Check workload status no longer shows deferred restarts.
|
||||
self.check_status_message_is_clear()
|
||||
|
||||
def run_show_deferred_events_action(self):
|
||||
"""Run show-deferred-events and return results.
|
||||
|
||||
:returns: Data from action run
|
||||
:rtype: Dict
|
||||
"""
|
||||
unit = model.get_units(self.application_name)[0]
|
||||
action = model.run_action(unit.entity_id, 'show-deferred-events')
|
||||
return yaml.safe_load(action.data['results']['output'])
|
||||
|
||||
def check_show_deferred_events_action_restart(self, test_service,
|
||||
restart_reason):
|
||||
"""Check the output from the action to list deferred restarts.
|
||||
|
||||
Run the action to list any deferred restarts and check it has entry for
|
||||
the given service and reason.
|
||||
|
||||
:param test_service: Service that should need a restart
|
||||
:type test_service: str
|
||||
:param restart_reason: The reason the action should list for the
|
||||
service needing to be restarted. This can be a
|
||||
substring.
|
||||
:type restart_reason: str
|
||||
"""
|
||||
# Ensure that the deferred restart and cause are listed via action
|
||||
logging.info(
|
||||
("Checking {} is marked as needing restart in "
|
||||
"show-deferred-events action").format(
|
||||
test_service))
|
||||
for event in self.run_show_deferred_events_action()['restarts']:
|
||||
logging.info("{} in {} and {} in {}".format(
|
||||
test_service,
|
||||
event,
|
||||
restart_reason,
|
||||
event))
|
||||
if test_service in event and restart_reason in event:
|
||||
break
|
||||
else:
|
||||
msg = 'No entry for restart of {} for reason {} found'.format(
|
||||
test_service,
|
||||
restart_reason)
|
||||
raise Exception(msg)
|
||||
|
||||
def check_show_deferred_events_action_hook(self, hook):
|
||||
"""Check the output from the action to list deferred eveents.
|
||||
|
||||
Run the action to list any deferred events and check it has entry for
|
||||
the given hook.
|
||||
|
||||
:param hook: Hook name
|
||||
:type hook: str
|
||||
"""
|
||||
# Ensure that the deferred restart and cause are listed via action
|
||||
logging.info(
|
||||
("Checking {} is marked as skipped in "
|
||||
"show-deferred-events action").format(hook))
|
||||
for event in self.run_show_deferred_events_action()['hooks']:
|
||||
logging.info("{} in {}".format(hook, event))
|
||||
if hook in event:
|
||||
break
|
||||
else:
|
||||
msg = '{} not found in {}'.format(hook, event)
|
||||
raise Exception(msg)
|
||||
|
||||
def check_show_deferred_restarts_wlm(self, test_service):
|
||||
"""Check the workload status message lists deferred restart.
|
||||
|
||||
:param test_service: Service that should need a restart
|
||||
:type test_service: str
|
||||
"""
|
||||
# Ensure that the deferred restarts are visible in Juju status
|
||||
for unit in model.get_units(self.application_name):
|
||||
# Just checking one example service should we be checking all?
|
||||
logging.info(
|
||||
("Checking {} is marked as needing restart in workload "
|
||||
"message of {}".format(test_service, unit.entity_id)))
|
||||
assert test_service in unit.workload_status_message
|
||||
|
||||
def check_deferred_hook_wlm(self):
|
||||
"""Check the workload status message lists deferred event."""
|
||||
# Ensure that the deferred restarts are visible in Juju status
|
||||
for unit in model.get_units(self.application_name):
|
||||
logging.info(
|
||||
("Checking {} is marked as having deferred hook in workload "
|
||||
"message".format(unit.entity_id)))
|
||||
assert 'config-changed' in unit.workload_status_message
|
||||
|
||||
def trigger_deferred_hook_via_charm(self):
|
||||
"""Set charm config option which requires a service start.
|
||||
|
||||
Set the charm debug option and wait for that change to be renderred in
|
||||
applications config file.
|
||||
|
||||
NOTE: The implementation assumes the charm has a `debug` option and
|
||||
self.restart_config_file in an oslo config file where that
|
||||
debug option is renderred. If that is not true the specaliasation
|
||||
class should override this method.
|
||||
"""
|
||||
app_config = model.get_application_config(self.application_name)
|
||||
logging.info("Triggering deferred restart via config change")
|
||||
new_debug_value = str(not app_config['debug']['value'])
|
||||
logging.info("Setting debug: {}".format(new_debug_value))
|
||||
model.set_application_config(
|
||||
self.application_name,
|
||||
{'debug': new_debug_value})
|
||||
logging.info("Waiting for units to be idle")
|
||||
test_unit = model.get_units(self.application_name)[0]
|
||||
model.block_until_unit_wl_message_match(
|
||||
test_unit.entity_id,
|
||||
status_pattern='.*config-changed.*')
|
||||
model.block_until_all_units_idle()
|
||||
|
||||
def trigger_deferred_restart_via_package(self):
|
||||
"""Update a package which requires a service restart."""
|
||||
logging.info("Triggering deferred restart via package change")
|
||||
# Test restart requested by package
|
||||
for unit in model.get_units(self.application_name):
|
||||
model.run_on_unit(
|
||||
unit.entity_id,
|
||||
'dpkg-reconfigure {}; ./hooks/update-status'.format(
|
||||
self.restart_package))
|
||||
|
||||
def run_charm_change_test(self):
|
||||
"""Trigger a deferred restart by updating a config file via the charm.
|
||||
|
||||
Trigger a config-changed hook in the charm which will update a config
|
||||
file and add a deferred restart.
|
||||
|
||||
NOTE: If this test is not relevant for the target charm then override
|
||||
this method and raise unittest.SkipTest
|
||||
"""
|
||||
self.trigger_deferred_hook_via_charm()
|
||||
|
||||
self.check_deferred_hook_wlm()
|
||||
self.check_show_deferred_events_action_hook('config-changed')
|
||||
# Rerunning to flip config option back to previous value.
|
||||
self.trigger_deferred_hook_via_charm()
|
||||
logging.info("Running restart action to clear deferred restarts")
|
||||
self.check_clear_hooks()
|
||||
|
||||
def run_package_change_test(self):
|
||||
"""Trigger a deferred restart by updating a package.
|
||||
|
||||
Update a package which requires will add a deferred restart.
|
||||
|
||||
NOTE: If this test is not relevant for the target charm then override
|
||||
this method and raise unittest.SkipTest
|
||||
"""
|
||||
self.trigger_deferred_restart_via_package()
|
||||
|
||||
self.check_show_deferred_restarts_wlm(self.restart_package_service)
|
||||
self.check_show_deferred_events_action_restart(
|
||||
self.restart_package_service,
|
||||
'Package update')
|
||||
logging.info("Running restart action to clear deferred restarts")
|
||||
self.check_clear_restarts()
|
||||
|
||||
def test_deferred_restarts(self):
|
||||
"""Run deferred restart tests."""
|
||||
app_config = model.get_application_config(self.application_name)
|
||||
auto_restart_config_key = 'enable-auto-restarts'
|
||||
if auto_restart_config_key not in app_config:
|
||||
raise unittest.SkipTest("Deferred restarts not implemented")
|
||||
|
||||
# Ensure auto restarts are off.
|
||||
policy_file = '/etc/policy-rc.d/charm-{}.policy'.format(
|
||||
self.application_name)
|
||||
if app_config[auto_restart_config_key]['value']:
|
||||
logging.info("Turning off auto restarts")
|
||||
model.set_application_config(
|
||||
self.application_name, {auto_restart_config_key: 'False'})
|
||||
logging.info("Waiting for {} to appear on units of {}".format(
|
||||
policy_file,
|
||||
self.application_name))
|
||||
model.block_until_file_has_contents(
|
||||
self.application_name,
|
||||
policy_file,
|
||||
'policy_requestor_name')
|
||||
# The block_until_file_has_contents ensures the change we waiting
|
||||
# for has happened, now just wait for any hooks to finish.
|
||||
logging.info("Waiting for units to be idle")
|
||||
model.block_until_all_units_idle()
|
||||
else:
|
||||
logging.info("Auto restarts already disabled")
|
||||
|
||||
# Trigger a config change which requires a restart
|
||||
self.run_charm_change_test()
|
||||
|
||||
# Trigger a package change which requires a restart
|
||||
self.run_package_change_test()
|
||||
|
||||
# Finished so turn auto-restarts back on.
|
||||
logging.info("Turning on auto restarts")
|
||||
model.set_application_config(
|
||||
self.application_name, {auto_restart_config_key: 'True'})
|
||||
model.block_until_file_missing(
|
||||
self.application_name,
|
||||
policy_file)
|
||||
model.block_until_all_units_idle()
|
||||
|
||||
Reference in New Issue
Block a user