From 1889d0db530ecc03f61976815d0704040b62e0e1 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Mon, 12 Oct 2020 17:51:04 +0200 Subject: [PATCH 1/5] Add func-tests for ceph-osd 'service' action --- zaza/openstack/charm_tests/ceph/osd/tests.py | 177 +++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index e3f7be1..b8080c0 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -16,6 +16,10 @@ import logging import unittest +import re + +from copy import deepcopy +from typing import List import zaza.openstack.charm_tests.test_utils as test_utils import zaza.model as zaza_model @@ -47,3 +51,176 @@ class SecurityTest(unittest.TestCase): expected_passes, expected_failures, expected_to_pass=True) + + +class OsdService: + """Simple representation of ceph-osd systemd service.""" + + def __init__(self, id_: int): + """ + Init service using its ID. + + e.g.: id_=1 -> ceph.-osd@1.service + """ + self.id = id_ + self.name = 'ceph-osd@{}'.format(id_) + + +class ServiceTest(unittest.TestCase): + """ceph-osd systemd service tests.""" + + TESTED_UNIT = 'ceph-osd/0' + SERVICE_PATTERN = re.compile(r'ceph-osd@(?P\d+)\.service') + + def __init__(self, methodName='runTest'): + """Initialize Test Case.""" + super(ServiceTest, self).__init__(methodName) + self._available_services = None + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph service tests.""" + super(ServiceTest, cls).setUpClass() + + def setUp(self): + """Run test setup.""" + # Note: This counter reset is needed because ceph-osd service is + # limited to 3 restarts per 30 mins which is insufficient + # when running functional tests for 'service' action. This + # limitation is defined in /lib/systemd/system/ceph-osd@.service + # in section [Service] with options 'StartLimitInterval' and + # 'StartLimitBurst' + reset_counter = 'systemctl reset-failed' + zaza_model.run_on_unit(self.TESTED_UNIT, reset_counter) + + def tearDown(self): + """Start ceph-osd services after each test. + + This ensures that the environment is ready for the next tests. + """ + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': 'all'}, + raise_on_failure=True) + + @property + def available_services(self) -> List[OsdService]: + """Return list of all ceph-osd services present on the TESTED_UNIT.""" + if self._available_services is None: + self._available_services = self._fetch_osd_services() + return self._available_services + + def _fetch_osd_services(self) -> List[OsdService]: + """Fetch all ceph-osd services present on the TESTED_UNIT.""" + service_list = [] + service_list_cmd = 'systemctl list-units --full --all ' \ + '--no-pager -t service' + result = zaza_model.run_on_unit(self.TESTED_UNIT, service_list_cmd) + for line in result['Stdout'].split('\n'): + service_name = self.SERVICE_PATTERN.search(line) + if service_name: + service_id = int(service_name.group('service_id')) + service_list.append(OsdService(service_id)) + return service_list + + def test_start_stop_all_by_keyword(self): + """Start and Stop all ceph-osd services using keyword 'all'.""" + service_list = [service.name for service in self.available_services] + + logging.info("Running 'service stop=all' action on {} " + "unit".format(self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'stop': 'all'}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') + + logging.info("Running 'service start=all' action on {} " + "unit".format(self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': 'all'}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running', + pgrep_full=True) + + def test_start_stop_all_by_list(self): + """Start and Stop all ceph-osd services using explicit list.""" + service_list = [service.name for service in self.available_services] + service_ids = [str(service.id) for service in self.available_services] + action_params = ','.join(service_ids) + + logging.info("Running 'service stop={}' action on {} " + "unit".format(action_params, self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'stop': action_params}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') + + logging.info("Running 'service start={}' action on {} " + "unit".format(action_params, self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': action_params}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running', + pgrep_full=True) + + def test_stop_specific(self): + """Stop only specified ceph-osd service.""" + if len(self.available_services) < 2: + raise unittest.SkipTest('This test can be performed only if ' + 'there\'s more than one ceph-osd service ' + 'present on the tested unit') + + should_run = deepcopy(self.available_services) + to_stop = should_run.pop() + should_run = [service.name for service in should_run] + + logging.info("Running 'service stop={} on {} " + "unit".format(to_stop.id, self.TESTED_UNIT)) + + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'stop': to_stop.id}) + + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=[to_stop.name, ], + target_status='stopped') + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=should_run, + target_status='running', + pgrep_full=True) + + def test_start_specific(self): + """Start only specified ceph-osd service.""" + if len(self.available_services) < 2: + raise unittest.SkipTest('This test can be performed only if ' + 'there\'s more than one ceph-osd service ' + 'present on the tested unit') + + service_names = [service.name for service in self.available_services] + should_stop = deepcopy(self.available_services) + to_start = should_stop.pop() + should_stop = [service.name for service in should_stop] + + logging.info("Stopping all running ceph-osd services") + service_stop_cmd = 'systemctl stop ceph-osd.target' + zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) + + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_names, + target_status='stopped') + + logging.info("Running 'service start={} on {} " + "unit".format(to_start.id, self.TESTED_UNIT)) + + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': to_start.id}) + + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=[to_start.name, ], + target_status='running', + pgrep_full=True) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=should_stop, + target_status='stopped') From 24755c23820167dbcc45b09eb0ec922da9f74a7c Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Wed, 14 Oct 2020 13:47:20 +0200 Subject: [PATCH 2/5] Add function that waits on desired service status using `systemctl` --- zaza/openstack/charm_tests/ceph/osd/tests.py | 108 +++++++++++++------ 1 file changed, 77 insertions(+), 31 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index b8080c0..58bc5bc 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -66,6 +66,55 @@ class OsdService: self.name = 'ceph-osd@{}'.format(id_) +async def async_wait_for_service_status(unit_name, services, target_status, + model_name=None, timeout=2700): + """Wait for all services on the unit are in the desired state. + + Note:This function emulates the + `zaza.model.async_block_until_service_status` function, but it's using + `systemctl is-active` command istead of `pidof/pgrep` of the original + function. + + :param unit_name: Name of unit to run action on + :type unit_name: str + :param services: List of services to check + :type services: [] + :param target_status: State services must be in (stopped or running) + :type target_status: str + :param model_name: Name of model to query. + :type model_name: str + :param timeout: Time to wait for status to be achieved + :type timeout: int + """ + async def _check_service(): + for service in services: + command = r"systemctl is-active '{}'".format(service) + out = await zaza_model.async_run_on_unit( + unit_name, + command, + model_name=model_name, + timeout=timeout) + response = out['Stdout'].strip() + + if target_status == "running" and response == 'active': + return True + elif target_status == "stopped" and response == 'inactive': + return True + else: + return False + logging.info("Unsing fancy method") + accepted_states = ('stopped', 'running') + if target_status not in accepted_states: + raise RuntimeError('Invalid target state "{}". Accepted states: ' + '{}'.format(target_status, accepted_states)) + + async with zaza_model.run_in_model(model_name): + await zaza_model.async_block_until(_check_service, timeout=timeout) + + +wait_for_service = zaza_model.sync_wrapper(async_wait_for_service_status) + + class ServiceTest(unittest.TestCase): """ceph-osd systemd service tests.""" @@ -130,18 +179,17 @@ class ServiceTest(unittest.TestCase): "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'stop': 'all'}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') logging.info("Running 'service start=all' action on {} " "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'start': 'all'}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='running', - pgrep_full=True) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running') def test_start_stop_all_by_list(self): """Start and Stop all ceph-osd services using explicit list.""" @@ -153,18 +201,17 @@ class ServiceTest(unittest.TestCase): "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'stop': action_params}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') logging.info("Running 'service start={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'start': action_params}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='running', - pgrep_full=True) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running') def test_stop_specific(self): """Stop only specified ceph-osd service.""" @@ -183,13 +230,12 @@ class ServiceTest(unittest.TestCase): zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'stop': to_stop.id}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=[to_stop.name, ], - target_status='stopped') - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=should_run, - target_status='running', - pgrep_full=True) + wait_for_service(unit_name=self.TESTED_UNIT, + services=[to_stop.name, ], + target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=should_run, + target_status='running') def test_start_specific(self): """Start only specified ceph-osd service.""" @@ -207,9 +253,9 @@ class ServiceTest(unittest.TestCase): service_stop_cmd = 'systemctl stop ceph-osd.target' zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_names, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_names, + target_status='stopped') logging.info("Running 'service start={} on {} " "unit".format(to_start.id, self.TESTED_UNIT)) @@ -217,10 +263,10 @@ class ServiceTest(unittest.TestCase): zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'start': to_start.id}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=[to_start.name, ], - target_status='running', - pgrep_full=True) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=should_stop, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=[to_start.name, ], + target_status='running') + + wait_for_service(unit_name=self.TESTED_UNIT, + services=should_stop, + target_status='stopped') From 12db27016eb348394cc8fd64d98924985da4b0a2 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Wed, 14 Oct 2020 13:55:23 +0200 Subject: [PATCH 3/5] Fix typos --- zaza/openstack/charm_tests/ceph/osd/tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 58bc5bc..35b6de4 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -68,11 +68,11 @@ class OsdService: async def async_wait_for_service_status(unit_name, services, target_status, model_name=None, timeout=2700): - """Wait for all services on the unit are in the desired state. + """Wait for all services on the unit to be in the desired state. - Note:This function emulates the + Note: This function emulates the `zaza.model.async_block_until_service_status` function, but it's using - `systemctl is-active` command istead of `pidof/pgrep` of the original + `systemctl is-active` command instead of `pidof/pgrep` of the original function. :param unit_name: Name of unit to run action on @@ -102,7 +102,7 @@ async def async_wait_for_service_status(unit_name, services, target_status, return True else: return False - logging.info("Unsing fancy method") + accepted_states = ('stopped', 'running') if target_status not in accepted_states: raise RuntimeError('Invalid target state "{}". Accepted states: ' From 990be121717396802a496de1fd8453de2ef67029 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Wed, 28 Oct 2020 13:58:49 +0100 Subject: [PATCH 4/5] Fix async_wait_for_service_status() checking only first service in the list --- zaza/openstack/charm_tests/ceph/osd/tests.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 35b6de4..6f1bfac 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -60,7 +60,7 @@ class OsdService: """ Init service using its ID. - e.g.: id_=1 -> ceph.-osd@1.service + e.g.: id_=1 -> ceph-osd@1 """ self.id = id_ self.name = 'ceph-osd@{}'.format(id_) @@ -87,6 +87,7 @@ async def async_wait_for_service_status(unit_name, services, target_status, :type timeout: int """ async def _check_service(): + services_ok = True for service in services: command = r"systemctl is-active '{}'".format(service) out = await zaza_model.async_run_on_unit( @@ -97,11 +98,14 @@ async def async_wait_for_service_status(unit_name, services, target_status, response = out['Stdout'].strip() if target_status == "running" and response == 'active': - return True + continue elif target_status == "stopped" and response == 'inactive': - return True + continue else: - return False + services_ok = False + break + + return services_ok accepted_states = ('stopped', 'running') if target_status not in accepted_states: From 991d7ea7a41fe86ed5c5abe36966214514c02cce Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Tue, 3 Nov 2020 15:47:18 +0100 Subject: [PATCH 5/5] Removed `typing` usage --- zaza/openstack/charm_tests/ceph/osd/tests.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 6f1bfac..03e212f 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -19,7 +19,6 @@ import unittest import re from copy import deepcopy -from typing import List import zaza.openstack.charm_tests.test_utils as test_utils import zaza.model as zaza_model @@ -56,7 +55,7 @@ class SecurityTest(unittest.TestCase): class OsdService: """Simple representation of ceph-osd systemd service.""" - def __init__(self, id_: int): + def __init__(self, id_): """ Init service using its ID. @@ -78,7 +77,7 @@ async def async_wait_for_service_status(unit_name, services, target_status, :param unit_name: Name of unit to run action on :type unit_name: str :param services: List of services to check - :type services: [] + :type services: List[str] :param target_status: State services must be in (stopped or running) :type target_status: str :param model_name: Name of model to query. @@ -122,7 +121,7 @@ wait_for_service = zaza_model.sync_wrapper(async_wait_for_service_status) class ServiceTest(unittest.TestCase): """ceph-osd systemd service tests.""" - TESTED_UNIT = 'ceph-osd/0' + TESTED_UNIT = 'ceph-osd/0' # This can be any ceph-osd unit in the model SERVICE_PATTERN = re.compile(r'ceph-osd@(?P\d+)\.service') def __init__(self, methodName='runTest'): @@ -156,13 +155,13 @@ class ServiceTest(unittest.TestCase): raise_on_failure=True) @property - def available_services(self) -> List[OsdService]: + def available_services(self): """Return list of all ceph-osd services present on the TESTED_UNIT.""" if self._available_services is None: self._available_services = self._fetch_osd_services() return self._available_services - def _fetch_osd_services(self) -> List[OsdService]: + def _fetch_osd_services(self): """Fetch all ceph-osd services present on the TESTED_UNIT.""" service_list = [] service_list_cmd = 'systemctl list-units --full --all ' \ @@ -181,7 +180,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop=all' action on {} " "unit".format(self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + zaza_model.run_action_on_units([self.TESTED_UNIT], 'service', action_params={'stop': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list,