From 74652f2523bd77d9411e7f08f0a97abe97faabd8 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 9 Apr 2020 17:29:05 +0200 Subject: [PATCH] Add unit test coverage for series upgrades This also includes some tidying --- ..._zaza_utilities_parallel_series_upgrade.py | 382 ++++++++++++++++++ .../utilities/parallel_series_upgrade.py | 59 +-- zaza/openstack/utilities/upgrade_utils.py | 4 +- 3 files changed, 406 insertions(+), 39 deletions(-) create mode 100644 unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py new file mode 100644 index 0000000..966b54e --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -0,0 +1,382 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import mock +import unit_tests.utils as ut_utils +import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.series_upgrade as series_upgrade +import zaza.openstack.utilities.parallel_series_upgrade as upgrade_utils + +FAKE_STATUS = { + 'can-upgrade-to': '', + 'charm': 'local:trusty/app-136', + 'subordinate-to': [], + 'units': {'app/0': {'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}, + 'app/1': {'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': {'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}} + + +class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase): + def setUp(self): + super(Test_ParallelSeriesUpgradeSync, self).setUp() + # Juju Status Object and data + # self.juju_status = mock.MagicMock() + # self.juju_status.applications.__getitem__.return_value = FAKE_STATUS + # self.patch_object(upgrade_utils, "model") + # self.model.get_status.return_value = self.juju_status + + def test_get_leader_and_non_leaders(self): + expected = ({ + 'app/0': { + 'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}}, { + 'app/1': { + 'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': { + 'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}) + + self.assertEqual( + expected, + upgrade_utils.get_leader_and_non_leaders(FAKE_STATUS) + ) + + def test_app_config_openstack_charm(self): + expected = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': False, } + config = upgrade_utils.app_config('keystone') + self.assertEqual(expected, config) + + def test_app_config_mongo(self): + expected = { + 'origin': None, + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': True, } + config = upgrade_utils.app_config('mongodb') + self.assertEqual(expected, config) + + def test_app_config_ceph(self): + expected = { + 'origin': 'source', + 'pause_non_leader_subordinate': False, + 'pause_non_leader_primary': False, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': False, } + config = upgrade_utils.app_config('ceph-mon') + self.assertEqual(expected, config) + + def test_app_config_percona(self): + expected = { + 'origin': 'source', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [ + ('zaza.openstack.charm_tests.mysql.utils.' + 'complete_cluster_series_upgrade') + ], + 'follower_first': False, } + config = upgrade_utils.app_config('percona-cluster') + self.assertEqual(expected, config) + + +class AioTestCase(ut_utils.BaseTestCase): + def __init__(self, methodName='runTest', loop=None): + self.loop = loop or asyncio.get_event_loop() + self._function_cache = {} + super(AioTestCase, self).__init__(methodName=methodName) + + def coroutine_function_decorator(self, func): + def wrapper(*args, **kw): + return self.loop.run_until_complete(func(*args, **kw)) + return wrapper + + def __getattribute__(self, item): + attr = object.__getattribute__(self, item) + if asyncio.iscoroutinefunction(attr) and item.startswith('test_'): + if item not in self._function_cache: + self._function_cache[item] = ( + self.coroutine_function_decorator(attr)) + return self._function_cache[item] + return attr + + +class TestParallelSeriesUpgrade(AioTestCase): + def setUp(self): + super(TestParallelSeriesUpgrade, self).setUp() + self.patch_object(series_upgrade, "async_prepare_series_upgrade") + self.patch_object(generic_utils, 'check_call') + # Juju Status Object and data + + self.juju_status = mock.AsyncMock() + self.juju_status.return_value.applications.__getitem__.return_value = \ + FAKE_STATUS + self.patch_object(upgrade_utils, "model") + self.model.async_get_status = self.juju_status + self.async_run_action = mock.AsyncMock() + self.model.async_run_action = self.async_run_action + + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_parallel_series_upgrade( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + mock_async_set_origin, + ): + await upgrade_utils.parallel_series_upgrade( + 'app', + from_series='trusty', + to_series='xenial', + ) + mock_async_set_series.assert_called_once_with( + 'app', to_series='xenial') + self.juju_status.assert_called() + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + mock.call('0', to_series='xenial'), + ]) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '1', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '2', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '0', + files=None, + workaround_script=None, + post_upgrade_functions=None), + ]) + mock_async_set_origin.assert_called_once_with( + 'app', 'openstack-origin') + mock_post_application_upgrade_functions.assert_called_once_with(None) + + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_serial_series_upgrade( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + mock_async_set_origin, + ): + await upgrade_utils.serial_series_upgrade( + 'app', + from_series='trusty', + to_series='xenial', + ) + mock_async_set_series.assert_called_once_with( + 'app', to_series='xenial') + self.juju_status.assert_called() + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('0', to_series='xenial'), + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + ]) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '0', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '1', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '2', + files=None, + workaround_script=None, + post_upgrade_functions=None), + ]) + mock_async_set_origin.assert_called_once_with( + 'app', 'openstack-origin') + mock_post_application_upgrade_functions.assert_called_once_with(None) + + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_complete_series_upgrade') + @mock.patch.object(upgrade_utils, 'reboot') + @mock.patch.object(upgrade_utils, 'async_do_release_upgrade') + @mock.patch.object(upgrade_utils, 'async_dist_upgrade') + async def test_series_upgrade_machine( + self, + mock_async_dist_upgrade, + mock_async_do_release_upgrade, + mock_reboot, + mock_async_complete_series_upgrade + ): + await upgrade_utils.series_upgrade_machine( + '1', + post_upgrade_functions=None, + pre_upgrade_functions=None, + files=None, + workaround_script=None) + mock_async_dist_upgrade.assert_called_once_with('1') + mock_async_do_release_upgrade.assert_called_once_with('1') + mock_reboot.assert_called_once_with('1') + mock_async_complete_series_upgrade.assert_called_once_with('1') + + async def test_maybe_pause_things_primary(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=False, + pause_non_leader_primary=True) + self.async_run_action.assert_has_calls([ + mock.call('app/1', "pause", action_params={}), + mock.call('app/2', "pause", action_params={}), + ]) + + async def test_maybe_pause_things_subordinates(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=True, + pause_non_leader_primary=False) + self.async_run_action.assert_has_calls([ + mock.call('app-hacluster/1', "pause", action_params={}), + mock.call('app-hacluster/2', "pause", action_params={}), + ]) + + async def test_maybe_pause_things_all(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=True, + pause_non_leader_primary=True) + self.async_run_action.assert_has_calls([ + mock.call('app-hacluster/1', "pause", action_params={}), + mock.call('app/1', "pause", action_params={}), + mock.call('app-hacluster/2', "pause", action_params={}), + mock.call('app/2', "pause", action_params={}), + ]) + + async def test_maybe_pause_things_none(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=False, + pause_non_leader_primary=False) + self.async_run_action.assert_not_called() + + @mock.patch.object(upgrade_utils, 'run_on_machine') + async def test_async_do_release_upgrade(self, mock_run_on_machine): + await upgrade_utils.async_do_release_upgrade('1') + do_release_upgrade_cmd = ( + 'yes | sudo DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive') + mock_run_on_machine.assert_called_once_with( + '1', do_release_upgrade_cmd, timeout='120m' + ) + + async def test_prepare_series_upgrade(self): + await upgrade_utils.prepare_series_upgrade( + '1', to_series='xenial' + ) + self.async_prepare_series_upgrade.assert_called_once_with( + '1', to_series='xenial' + ) + + @mock.patch.object(upgrade_utils, 'run_on_machine') + async def test_reboot(self, mock_run_on_machine): + await upgrade_utils.reboot('1') + mock_run_on_machine.assert_called_once_with( + '1', 'shutdown --reboot now & exit' + ) + + async def test_run_on_machine(self): + await upgrade_utils.run_on_machine('1', 'test') + self.check_call.assert_called_once_with( + ['juju', 'run', '--machine=1', 'test']) + + async def test_run_on_machine_with_timeout(self): + await upgrade_utils.run_on_machine('1', 'test', timeout='20m') + self.check_call.assert_called_once_with( + ['juju', 'run', '--machine=1', '--timeout=20m', 'test']) + + async def test_run_on_machine_with_model(self): + await upgrade_utils.run_on_machine('1', 'test', model_name='test') + self.check_call.assert_called_once_with( + ['juju', 'run', '--machine=1', '--model=test', 'test']) + + @mock.patch.object(upgrade_utils, 'run_on_machine') + async def test_async_dist_upgrade(self, mock_run_on_machine): + await upgrade_utils.async_dist_upgrade('1') + apt_update_command = ( + """yes | sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + mock_run_on_machine.assert_has_calls([ + mock.call('1', 'sudo apt update'), + mock.call('1', apt_update_command), + ]) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 87deb51..7e6693e 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -187,7 +187,9 @@ async def parallel_series_upgrade( .format(application, follower_first)) status = (await model.async_get_status()).applications[application] - leader, non_leaders = await get_leader_and_non_leaders(status, application) + logging.info( + "Configuring leader / non leaders for {}".format(application)) + leader, non_leaders = get_leader_and_non_leaders(status) for leader_name, leader_unit in leader.items(): leader_machine = leader_unit["machine"] leader = leader_name @@ -205,10 +207,12 @@ async def parallel_series_upgrade( application, to_series=to_series) prepare_group = [ - prepare_series_upgrade(machine, to_series=to_series) + series_upgrade_utils.async_prepare_series_upgrade( + machine, to_series=to_series) for machine in machines] asyncio.gather(*prepare_group) - await prepare_series_upgrade(leader_machine, to_series=to_series) + await series_upgrade_utils.async_prepare_series_upgrade( + leader_machine, to_series=to_series) if leader_machine not in completed_machines: machines.append(leader_machine) upgrade_group = [ @@ -285,7 +289,9 @@ async def serial_series_upgrade( "About to upgrade the units of {} in serial (follower first: {})" .format(application, follower_first)) status = (await model.async_get_status()).applications[application] - leader, non_leaders = await get_leader_and_non_leaders(application) + logging.info( + "Configuring leader / non leaders for {}".format(application)) + leader, non_leaders = get_leader_and_non_leaders(status) for leader_name, leader_unit in leader.items(): leader_machine = leader_unit["machine"] leader = leader_name @@ -304,7 +310,8 @@ async def serial_series_upgrade( application, to_series=to_series) if not follower_first and leader_machine not in completed_machines: - await prepare_series_upgrade(leader_machine, to_series) + await series_upgrade_utils.async_prepare_series_upgrade( + leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) await series_upgrade_machine( @@ -313,7 +320,8 @@ async def serial_series_upgrade( post_upgrade_functions=post_upgrade_functions) for machine in machines: - await prepare_series_upgrade(machine, to_series) + await series_upgrade_utils.async_prepare_series_upgrade( + machine, to_series=to_series) logging.info("About to upgrade follower of {}: {}" .format(application, machine)) await series_upgrade_machine( @@ -322,7 +330,8 @@ async def serial_series_upgrade( post_upgrade_functions=post_upgrade_functions) if follower_first and leader_machine not in completed_machines: - await prepare_series_upgrade(leader_machine, to_series) + await series_upgrade_utils.async_prepare_series_upgrade( + leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) await series_upgrade_machine( @@ -360,7 +369,7 @@ async def series_upgrade_machine( logging.info( "About to dist-upgrade ({})".format(machine)) - run_pre_upgrade_functions(pre_upgrade_functions) + run_pre_upgrade_functions(machine, pre_upgrade_functions) await async_dist_upgrade(machine) await async_do_release_upgrade(machine) await reboot(machine) @@ -436,7 +445,7 @@ async def maybe_pause_things( await asyncio.gather(*leader_pauses) -async def get_leader_and_non_leaders(status, application): +def get_leader_and_non_leaders(status): """Get the leader and non-leader Juju units. This function returns a tuple that looks like: @@ -452,13 +461,9 @@ async def get_leader_and_non_leaders(status, application): The first entry of this tuple is the leader, and the second is all non-leader units. - :param application: Application to fetch details for - :type application: str :returns: A tuple of dicts identifying leader and non-leaders :rtype: Dict[str, List[juju.Unit]] """ - logging.info( - "Configuring leader / non leaders for {}".format(application)) leader = None non_leaders = {} for name, unit in status["units"].items(): @@ -487,42 +492,22 @@ async def prepare_series_upgrade(machine, to_series): machine, to_series=to_series) -async def reboot(unit): +async def reboot(machine): """Reboot the named machine. - :param unit: Machine to reboot - :type unit: str + :param machine: Machine to reboot + :type machine: str :returns: Nothing :rtype: None """ try: - await run_on_machine(unit, 'shutdown --reboot now & exit') + await run_on_machine(machine, 'shutdown --reboot now & exit') # await run_on_machine(unit, "sudo reboot && exit") except subprocess.CalledProcessError as e: logging.warn("Error doing reboot: {}".format(e)) pass -async def complete_series_upgrade(machines, to_series): - """Execute juju series-upgrade complete on machine. - - NOTE: This is a new feature in juju behind a feature flag and not yet in - libjuju. - export JUJU_DEV_FEATURE_FLAGS=upgrade-series - :param machine_num: Machine number - :type machine_num: str - :returns: None - :rtype: None - """ - logging.info("Completing series upgrade for {}".format(machines)) - group = [] - for machine in machines: - # This can fail on the non-leaders if the leader goes first? - group.append( - series_upgrade_utils.async_complete_series_upgrade(machine)) - await asyncio.gather(*group) - - async def run_on_machine(machine, command, model_name=None, timeout=None): """Juju run on unit. diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index c864b9b..4066d19 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -20,8 +20,8 @@ import zaza.model SERVICE_GROUPS = collections.OrderedDict([ - ('Stateful Services', ['percona-cluster', 'rabbitmq-server']), - ('Core Identity', ['keystone', 'ceph-mon']), + ('Stateful Services', ['percona-cluster', 'rabbitmq-server', 'ceph-mon']), + ('Core Identity', ['keystone']), ('Control Plane', [ 'aodh', 'barbican', 'ceilometer', 'ceph-fs', 'ceph-radosgw', 'cinder', 'designate',