arif/zaza-openstack-tests
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Deferred restart test fixes (#540)

Browse Source 
* Add ovn-chassis test
* Yet another refactor to reduce the amount that implementations need to override.
* Add ovn dedicated chassis support
* Fix race with checking wlm
This commit is contained in:
Liam Young
2021-04-09 10:48:52 +01:00
committed by GitHub
parent 877deb223b
commit 01544d22ed
4 changed files with 291 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
zaza/openstack/charm_tests/neutron/tests.py
View File

@@ -1013,9 +1013,35 @@ class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest):
@classmethod
def setUpClass(cls):
"""Run setup for deferred restart tests."""
super(NeutronOVSDeferredRestartTest, cls).setUpClass(
restart_config_file='/etc/neutron/neutron.conf',
test_service='neutron-openvswitch-agent',
restart_package='openvswitch-switch',
restart_package_service='openvswitch-switch',
application_name='neutron-openvswitch')
super().setUpClass(application_name='neutron-openvswitch')
def run_tests(self):
"""Run deferred restart tests."""
# Trigger a config change which triggers a deferred hook.
self.run_charm_change_hook_test('config-changed')
# Trigger a package change which requires a restart
self.run_package_change_test(
'openvswitch-switch',
'openvswitch-switch')
class NeutronGatewayDeferredRestartTest(test_utils.BaseDeferredRestartTest):
"""Deferred restart tests."""
@classmethod
def setUpClass(cls):
"""Run setup for deferred restart tests."""
super().setUpClass(application_name='neutron-gateway')
def run_tests(self):
"""Run deferred restart tests."""
# Trigger a config change which requires a restart
self.run_charm_change_restart_test(
'neutron-l3-agent',
'/etc/neutron/neutron.conf')
# Trigger a package change which requires a restart
self.run_package_change_test(
'openvswitch-switch',
'openvswitch-switch')

58
zaza/openstack/charm_tests/ovn/tests.py
View File

@@ -431,3 +431,61 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest):
except KeyError:
# One of the applications is not in the model, which is fine
pass
class OVNChassisDeferredRestartTest(test_utils.BaseDeferredRestartTest):
"""Deferred restart tests."""
@classmethod
def setUpClass(cls):
"""Run setup for deferred restart tests."""
super().setUpClass(application_name='ovn-chassis')
def run_tests(self):
"""Run deferred restart tests."""
# Trigger a config change which triggers a deferred hook.
self.run_charm_change_hook_test('configure_ovs')
# Trigger a package change which requires a restart
self.run_package_change_test(
'openvswitch-switch',
'openvswitch-switch')
def get_new_config(self):
"""Return the config key and new value to trigger a hook execution.
:returns: Config key and new value
:rtype: (str, bool)
"""
app_config = zaza.model.get_application_config(self.application_name)
return 'enable-sriov', str(not app_config['enable-sriov']['value'])
class OVNDedicatedChassisDeferredRestartTest(
test_utils.BaseDeferredRestartTest):
"""Deferred restart tests."""
@classmethod
def setUpClass(cls):
"""Run setup for deferred restart tests."""
super().setUpClass(application_name='ovn-dedicated-chassis')
def run_tests(self):
"""Run deferred restart tests."""
# Trigger a config change which triggers a deferred hook.
self.run_charm_change_hook_test('configure_ovs')
# Trigger a package change which requires a restart
self.run_package_change_test(
'openvswitch-switch',
'openvswitch-switch')
def get_new_config(self):
"""Return the config key and new value to trigger a hook execution.
:returns: Config key and new value
:rtype: (str, bool)
"""
app_config = zaza.model.get_application_config(self.application_name)
new_value = str(not app_config['disable-mlockall']['value'])
return 'disable-mlockall', new_value

68
zaza/openstack/charm_tests/rabbitmq_server/tests.py
View File

@@ -436,40 +436,50 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest):
@classmethod
def setUpClass(cls):
"""Run setup for deferred restart tests."""
super().setUpClass(
restart_config_file='/etc/rabbitmq/rabbitmq.config',
test_service='rabbitmq-server',
restart_package='rabbitmq-server',
restart_package_service='rabbitmq-server',
application_name='rabbitmq-server')
super().setUpClass(application_name='rabbitmq-server')
def check_status_message_is_clear(self):
"""Check each units status message show no defeerred events."""
pattern = '(Unit is ready|Unit is ready and clustered)$'
for unit in zaza.model.get_units(self.application_name):
assert unit.workload_status_message in [
'Unit is ready',
'Unit is ready and clustered']
zaza.model.block_until_unit_wl_message_match(
unit.entity_id,
pattern)
zaza.model.block_until_all_units_idle()
def trigger_deferred_hook_via_charm(self):
"""Set charm config option which requires a service start.
def get_new_config(self):
"""Return the config key and new value to trigger a hook execution.
Set the charm debug option and wait for that change to be renderred in
applications config file.
This overrides the base class version as the rabbit charm does not
have a debug option and the config file is not in oslo config format.
:returns: Config key and new value
:rtype: (str, bool)
"""
app_config = zaza.model.get_application_config(self.application_name)
logging.info("Triggering deferred restart via config change")
new_debug_value = str(
int(app_config['connection-backlog'].get('value', 100) + 1))
logging.info("Setting connection-backlog: {}".format(new_debug_value))
zaza.model.set_application_config(
self.application_name,
{'connection-backlog': new_debug_value})
logging.info("Waiting for units to be idle")
test_unit = zaza.model.get_units(self.application_name)[0]
zaza.model.block_until_unit_wl_message_match(
test_unit.entity_id,
status_pattern='.*config-changed.*')
zaza.model.block_until_all_units_idle()
new_value = str(int(
app_config['connection-backlog'].get('value', 100) + 1))
return 'connection-backlog', new_value
def run_tests(self):
"""Run deferred restart tests."""
# Trigger a config change which triggers a deferred hook.
self.run_charm_change_hook_test('config-changed')
# Trigger a package change which requires a restart
self.run_package_change_test(
'rabbitmq-server',
'rabbitmq-server')
def check_clear_restarts(self):
"""Clear and deferred restarts and check status.
Clear and deferred restarts and then check the workload status message
for each unit.
"""
# Use action to run any deferred restarts
for unit in zaza.model.get_units(self.application_name):
zaza.model.run_action(
unit.entity_id,
'restart-services',
action_params={'services': 'rabbitmq-server'})
# Check workload status no longer shows deferred restarts.
self.check_status_message_is_clear()

235
zaza/openstack/charm_tests/test_utils.py
View File

@@ -704,7 +704,7 @@ class OpenStackBaseTest(BaseCharmTest):
return instance_1, instance_2
class BaseDeferredRestartTest(OpenStackBaseTest):
class BaseDeferredRestartTest(BaseCharmTest):
"""Check deferred restarts.
Example of adding a deferred restart test::
@@ -714,12 +714,17 @@ class BaseDeferredRestartTest(OpenStackBaseTest):
@classmethod
def setUpClass(cls):
super(NeutronOVSDeferredRestartTest, cls).setUpClass(
restart_config_file = '/etc/neutron/neutron.conf',
test_service = 'neutron-openvswitch-agent',
restart_package = 'openvswitch-switch',
restart_package_service = 'openvswitch-switch',
application_name = 'neutron-openvswitch')
super().setUpClass(application_name='neutron-openvswitch')
def run_tests(self):
# Trigger a config change which triggers a deferred hook.
self.run_charm_change_hook_test('config-changed')
# Trigger a package change which requires a restart
self.run_package_change_test(
'openvswitch-switch',
'openvswitch-switch')
NOTE: The test has been broken into various class methods which may require
specialisation if the charm being tested is not a standard OpenStack
@@ -728,38 +733,23 @@ class BaseDeferredRestartTest(OpenStackBaseTest):
"""
@classmethod
def setUpClass(cls, restart_config_file, test_service, restart_package,
restart_package_service, application_name):
def setUpClass(cls, application_name):
"""Run test setup.
:param restart_config_file: Config file that will be changed to trigger
a service restart.
:type restart_config_file: str
:param test_service: Service that will require a restart after
restart_config_file has changed.
:type test_service: str
:param restart_package: Package that will be changed to trigger a
service restart.
:type restart_package: str
:param restart_package_service: Service that will require a restart
after restart_package has changed.
:type restart_package_service: str
:param application_name: Name of application to run tests against.
:type application_name: str
"""
cls.restart_config_file = restart_config_file
cls.test_service = test_service
cls.restart_package = restart_package
cls.restart_package_service = restart_package_service
cls.application_name = application_name
super(BaseDeferredRestartTest, cls).setUpClass(
application_name=cls.application_name)
super().setUpClass(application_name=cls.application_name)
def check_status_message_is_clear(self):
"""Check each units status message show no defeerred events."""
# Check workload status no longer shows deferred restarts.
for unit in model.get_units(self.application_name):
assert unit.workload_status_message == 'Unit is ready'
model.block_until_unit_wl_message_match(
unit.entity_id,
'Unit is ready')
model.block_until_all_units_idle()
def check_clear_restarts(self):
"""Clear and deferred restarts and check status.
@@ -772,7 +762,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest):
model.run_action(
unit.entity_id,
'restart-services',
action_params={'deferred': True})
action_params={'deferred-only': True})
# Check workload status no longer shows deferred restarts.
self.check_status_message_is_clear()
@@ -841,7 +831,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest):
Run the action to list any deferred events and check it has entry for
the given hook.
:param hook: Hook name
:param hook: Hook or method name
:type hook: str
"""
# Ensure that the deferred restart and cause are listed via action
@@ -870,85 +860,188 @@ class BaseDeferredRestartTest(OpenStackBaseTest):
"message of {}".format(test_service, unit.entity_id)))
assert test_service in unit.workload_status_message
def check_deferred_hook_wlm(self):
"""Check the workload status message lists deferred event."""
def check_deferred_hook_wlm(self, deferred_hook):
"""Check the workload status message lists deferred event.
:param deferred_hook: Hook or method name which should be showing as
deferred.
:type deferred_hook: str
"""
# Ensure that the deferred restarts are visible in Juju status
for unit in model.get_units(self.application_name):
logging.info(
("Checking {} is marked as having deferred hook in workload "
"message".format(unit.entity_id)))
assert 'config-changed' in unit.workload_status_message
assert deferred_hook in unit.workload_status_message
def trigger_deferred_hook_via_charm(self):
def get_new_config(self):
"""Return the config key and new value to trigger a hook execution.
NOTE: The implementation assumes the charm has a `debug` option and
If that is not true the derived class should override this
method.
:returns: Config key and new value
:rtype: (str, bool)
"""
app_config = model.get_application_config(self.application_name)
return 'debug', str(not app_config['debug']['value'])
def set_new_config(self):
"""Change applications charm config."""
logging.info("Triggering deferred restart via config change")
config_key, new_value = self.get_new_config()
logging.info("Setting {}: {}".format(config_key, new_value))
model.set_application_config(
self.application_name,
{config_key: new_value})
return new_value
def trigger_deferred_restart_via_charm(self, restart_config_file):
"""Set charm config option which requires a service start.
Set the charm debug option and wait for that change to be renderred in
applications config file.
NOTE: The implementation assumes the charm has a `debug` option and
self.restart_config_file in an oslo config file where that
debug option is renderred. If that is not true the specaliasation
class should override this method.
NOTE: The implementation assumes the restart_config_file in an oslo
config file. If that is not true the derived class should
override this method.
:param restart_config_file: Config file that updated value is expected
in.
:type restart_config_file: str
"""
app_config = model.get_application_config(self.application_name)
logging.info("Triggering deferred restart via config change")
new_debug_value = str(not app_config['debug']['value'])
logging.info("Setting debug: {}".format(new_debug_value))
model.set_application_config(
new_debug_value = self.set_new_config()
expected_contents = {
'DEFAULT': {
'debug': [new_debug_value]}}
logging.info("Waiting for debug to be {} in {}".format(
new_debug_value,
restart_config_file))
model.block_until_oslo_config_entries_match(
self.application_name,
{'debug': new_debug_value})
restart_config_file,
expected_contents)
logging.info("Waiting for units to be idle")
test_unit = model.get_units(self.application_name)[0]
model.block_until_unit_wl_message_match(
test_unit.entity_id,
status_pattern='.*config-changed.*')
model.block_until_all_units_idle()
def trigger_deferred_restart_via_package(self):
"""Update a package which requires a service restart."""
def trigger_deferred_hook_via_charm(self, deferred_hook):
"""Set charm config option which requires a service start.
Set the charm debug option and wait for that change to be rendered in
applications config file.
:param deferred_hook: Hook or method name which should be showing as
deferred.
:type deferred_hook: str
:returns: New config value
:rtype: Union[str, int, float]
"""
new_debug_value = self.set_new_config()
for unit in model.get_units(self.application_name):
logging.info('Waiting for {} to show deferred hook'.format(
unit.entity_id))
model.block_until_unit_wl_message_match(
unit.entity_id,
status_pattern='.*{}.*'.format(deferred_hook))
logging.info("Waiting for units to be idle")
model.block_until_all_units_idle()
return new_debug_value
def trigger_deferred_restart_via_package(self, restart_package):
"""Update a package which requires a service restart.
:param restart_package: Package that will be changed to trigger a
service restart.
:type restart_package: str
"""
logging.info("Triggering deferred restart via package change")
# Test restart requested by package
for unit in model.get_units(self.application_name):
model.run_on_unit(
unit.entity_id,
'dpkg-reconfigure {}; ./hooks/update-status'.format(
self.restart_package))
('dpkg-reconfigure {}; '
'JUJU_HOOK_NAME=update-status ./hooks/update-status').format(
restart_package))
def run_charm_change_test(self):
def run_charm_change_restart_test(self, test_service, restart_config_file):
"""Trigger a deferred restart by updating a config file via the charm.
Trigger a config-changed hook in the charm which will update a config
file and add a deferred restart.
Trigger a hook in the charm which the charm will defer.
NOTE: If this test is not relevant for the target charm then override
this method and raise unittest.SkipTest
:param test_service: Service that should need a restart
:type test_service: str
:param restart_config_file: Config file that updated value is expected
in.
:type restart_config_file: str
"""
self.trigger_deferred_hook_via_charm()
self.trigger_deferred_restart_via_charm(restart_config_file)
self.check_deferred_hook_wlm()
self.check_show_deferred_events_action_hook('config-changed')
# Rerunning to flip config option back to previous value.
self.trigger_deferred_hook_via_charm()
self.check_show_deferred_restarts_wlm(test_service)
self.check_show_deferred_events_action_restart(
test_service,
restart_config_file)
logging.info("Running restart action to clear deferred restarts")
self.check_clear_restarts()
def run_charm_change_hook_test(self, deferred_hook):
"""Trigger a deferred restart by updating a config file via the charm.
:param deferred_hook: Hook or method name which should be showing as
defeerred.
:type deferred_hook: str
"""
self.trigger_deferred_hook_via_charm(deferred_hook)
self.check_deferred_hook_wlm(deferred_hook)
self.check_show_deferred_events_action_hook(deferred_hook)
# Rerunning to flip config option back to previous value.
self.trigger_deferred_hook_via_charm(deferred_hook)
logging.info("Running restart action to clear deferred hooks")
self.check_clear_hooks()
def run_package_change_test(self):
def run_package_change_test(self, restart_package, restart_package_svc):
"""Trigger a deferred restart by updating a package.
Update a package which requires will add a deferred restart.
NOTE: If this test is not relevant for the target charm then override
this method and raise unittest.SkipTest
:param restart_package: Package that will be changed to trigger a
service restart.
:type restart_package: str
:param restart_package_service: Service that will require a restart
after restart_package has changed.
:type restart_package_service: str
"""
self.trigger_deferred_restart_via_package()
self.trigger_deferred_restart_via_package(restart_package)
self.check_show_deferred_restarts_wlm(self.restart_package_service)
self.check_show_deferred_restarts_wlm(restart_package_svc)
self.check_show_deferred_events_action_restart(
self.restart_package_service,
restart_package_svc,
'Package update')
logging.info("Running restart action to clear deferred restarts")
self.check_clear_restarts()
def run_tests(self):
"""Run charm tests. should specify which tests to run.
The charm test that implements this test should specify which tests to
run, for example:
def run_tests(self):
# Trigger a config change which triggers a deferred hook.
self.run_charm_change_hook_test('config-changed')
# Trigger a config change which requires a restart
self.run_charm_change_restart_test(
'neutron-l3-agent',
'/etc/neutron/neutron.conf')
# Trigger a package change which requires a restart
self.run_package_change_test(
'openvswitch-switch',
'openvswitch-switch')
"""
raise NotImplementedError
def test_deferred_restarts(self):
"""Run deferred restart tests."""
app_config = model.get_application_config(self.application_name)
@@ -977,11 +1070,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest):
else:
logging.info("Auto restarts already disabled")
# Trigger a config change which requires a restart
self.run_charm_change_test()
# Trigger a package change which requires a restart
self.run_package_change_test()
self.run_tests()
# Finished so turn auto-restarts back on.
logging.info("Turning on auto restarts")