diff --git a/tox.ini b/tox.ini index 6a37cfe..7d67e9f 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ setenv = VIRTUAL_ENV={envdir} install_command = pip install {opts} {packages} -commands = nosetests --with-coverage --cover-package=zaza {posargs} {toxinidir}/unit_tests +commands = nosetests --with-coverage --cover-package=zaza.openstack {posargs} {toxinidir}/unit_tests [testenv:py3] basepython = python3 diff --git a/unit_tests/charm_tests/__init__.py b/unit_tests/charm_tests/__init__.py new file mode 100644 index 0000000..6131624 --- /dev/null +++ b/unit_tests/charm_tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/unit_tests/charm_tests/test_mysql.py b/unit_tests/charm_tests/test_mysql.py new file mode 100644 index 0000000..e38460b --- /dev/null +++ b/unit_tests/charm_tests/test_mysql.py @@ -0,0 +1,40 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import mock +import unittest +import sys + +import zaza.openstack.charm_tests.mysql.utils as mysql_utils + + +class TestMysqlUtils(unittest.TestCase): + """Test class to encapsulate testing Mysql test utils.""" + + def setUp(self): + super(TestMysqlUtils, self).setUp() + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + + @mock.patch.object(mysql_utils, 'model') + def test_mysql_complete_cluster_series_upgrade(self, mock_model): + run_action_on_leader = mock.AsyncMock() + mock_model.async_run_action_on_leader = run_action_on_leader + asyncio.get_event_loop().run_until_complete( + mysql_utils.complete_cluster_series_upgrade()) + run_action_on_leader.assert_called_once_with( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/unit_tests/charm_tests/test_rabbitmq_server.py b/unit_tests/charm_tests/test_rabbitmq_server.py new file mode 100644 index 0000000..e092122 --- /dev/null +++ b/unit_tests/charm_tests/test_rabbitmq_server.py @@ -0,0 +1,40 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import mock +import unittest +import sys + +import zaza.openstack.charm_tests.rabbitmq_server.utils as rabbit_utils + + +class TestRabbitUtils(unittest.TestCase): + """Test class to encapsulate testing Mysql test utils.""" + + def setUp(self): + super(TestRabbitUtils, self).setUp() + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + + @mock.patch.object(rabbit_utils.zaza, 'model') + def test_rabbit_complete_cluster_series_upgrade(self, mock_model): + run_action_on_leader = mock.AsyncMock() + mock_model.async_run_action_on_leader = run_action_on_leader + asyncio.get_event_loop().run_until_complete( + rabbit_utils.complete_cluster_series_upgrade()) + run_action_on_leader.assert_called_once_with( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py new file mode 100644 index 0000000..aefa00b --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -0,0 +1,559 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import mock +import sys +import unittest +import unit_tests.utils as ut_utils +import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.series_upgrade as series_upgrade +import zaza.openstack.utilities.parallel_series_upgrade as upgrade_utils + +FAKE_STATUS = { + 'can-upgrade-to': '', + 'charm': 'local:trusty/app-136', + 'subordinate-to': [], + 'units': {'app/0': {'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}, + 'app/1': {'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': {'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}} + +FAKE_STATUS_MONGO = { + 'can-upgrade-to': '', + 'charm': 'local:trusty/mongodb-10', + 'subordinate-to': [], + 'units': {'mongo/0': {'leader': True, + 'machine': '0', + 'subordinates': {}}, + 'mongo/1': {'machine': '1', + 'subordinates': {}}, + 'mongo/2': {'machine': '2', + 'subordinates': {}}}} + + +class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase): + def setUp(self): + super(Test_ParallelSeriesUpgradeSync, self).setUp() + # Juju Status Object and data + # self.juju_status = mock.MagicMock() + # self.juju_status.applications.__getitem__.return_value = FAKE_STATUS + # self.patch_object(upgrade_utils, "model") + # self.model.get_status.return_value = self.juju_status + + def test_get_leader_and_non_leaders(self): + expected = ({ + 'app/0': { + 'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}}, { + 'app/1': { + 'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': { + 'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}) + + self.assertEqual( + expected, + upgrade_utils.get_leader_and_non_leaders(FAKE_STATUS) + ) + + def test_app_config_openstack_charm(self): + expected = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': False, } + config = upgrade_utils.app_config('keystone') + self.assertEqual(expected, config) + + def test_app_config_mongo(self): + expected = { + 'origin': None, + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': True, } + config = upgrade_utils.app_config('mongodb') + self.assertEqual(expected, config) + + def test_app_config_ceph(self): + expected = { + 'origin': 'source', + 'pause_non_leader_subordinate': False, + 'pause_non_leader_primary': False, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': False, } + config = upgrade_utils.app_config('ceph-mon') + self.assertEqual(expected, config) + + def test_app_config_percona(self): + expected = { + 'origin': 'source', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [ + ('zaza.openstack.charm_tests.mysql.utils.' + 'complete_cluster_series_upgrade') + ], + 'follower_first': False, } + config = upgrade_utils.app_config('percona-cluster') + self.assertEqual(expected, config) + + +class AioTestCase(ut_utils.BaseTestCase): + def __init__(self, methodName='runTest', loop=None): + self.loop = loop or asyncio.get_event_loop() + self._function_cache = {} + super(AioTestCase, self).__init__(methodName=methodName) + + def coroutine_function_decorator(self, func): + def wrapper(*args, **kw): + return self.loop.run_until_complete(func(*args, **kw)) + return wrapper + + def __getattribute__(self, item): + attr = object.__getattribute__(self, item) + if asyncio.iscoroutinefunction(attr) and item.startswith('test_'): + if item not in self._function_cache: + self._function_cache[item] = ( + self.coroutine_function_decorator(attr)) + return self._function_cache[item] + return attr + + +class TestParallelSeriesUpgrade(AioTestCase): + def setUp(self): + super(TestParallelSeriesUpgrade, self).setUp() + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + self.patch_object(series_upgrade, "async_prepare_series_upgrade") + self.patch_object(generic_utils, 'check_call') + # Juju Status Object and data + + self.juju_status = mock.AsyncMock() + self.juju_status.return_value.applications.__getitem__.return_value = \ + FAKE_STATUS + self.patch_object(upgrade_utils, "model") + self.model.async_get_status = self.juju_status + self.async_run_action = mock.AsyncMock() + self.model.async_run_action = self.async_run_action + + self.async_block_until = mock.AsyncMock() + self.model.async_block_until = self.async_block_until + self.model.async_wait_for_unit_idle = mock.AsyncMock() + self.async_run_on_machine = mock.AsyncMock() + self.model.async_run_on_machine = self.async_run_on_machine + + @mock.patch.object(upgrade_utils.cl_utils, 'get_class') + async def test_run_post_application_upgrade_functions( + self, + mock_get_class + ): + called = mock.AsyncMock() + mock_get_class.return_value = called + await upgrade_utils.run_post_application_upgrade_functions( + ['my.thing']) + mock_get_class.assert_called_once_with('my.thing') + called.assert_called() + + @mock.patch.object(upgrade_utils.cl_utils, 'get_class') + async def test_run_pre_upgrade_functions(self, mock_get_class): + called = mock.AsyncMock() + mock_get_class.return_value = called + await upgrade_utils.run_pre_upgrade_functions('1', ['my.thing']) + mock_get_class.assert_called_once_with('my.thing') + called.assert_called_once_with('1') + + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_parallel_series_upgrade_mongo( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + ): + self.juju_status.return_value.applications.__getitem__.return_value = \ + FAKE_STATUS_MONGO + upgrade_config = upgrade_utils.app_config('mongodb') + await upgrade_utils.parallel_series_upgrade( + 'mongodb', + from_series='trusty', + to_series='xenial', + **upgrade_config + ) + mock_async_set_series.assert_called_once_with( + 'mongodb', to_series='xenial') + self.juju_status.assert_called() + + # The below is using `any_order=True` because the ordering is + # undetermined and differs between python versions + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + mock.call('0', to_series='xenial'), + ], any_order=True) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '1', + origin=None, + application='mongodb', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + mock.call( + '2', + origin=None, + application='mongodb', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + mock.call( + '0', + origin=None, + application='mongodb', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + ]) + mock_post_application_upgrade_functions.assert_called_once_with([]) + + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_serial_series_upgrade_mongo( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + ): + self.juju_status.return_value.applications.__getitem__.return_value = \ + FAKE_STATUS_MONGO + upgrade_config = upgrade_utils.app_config('mongodb') + await upgrade_utils.serial_series_upgrade( + 'mongodb', + from_series='trusty', + to_series='xenial', + **upgrade_config + ) + mock_async_set_series.assert_called_once_with( + 'mongodb', to_series='xenial') + self.juju_status.assert_called() + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + mock.call('0', to_series='xenial'), + ]) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '1', + origin=None, + application='mongodb', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + mock.call( + '2', + origin=None, + application='mongodb', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + mock.call( + '0', + origin=None, + application='mongodb', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + ]) + mock_post_application_upgrade_functions.assert_called_once_with([]) + + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_parallel_series_upgrade( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + ): + await upgrade_utils.parallel_series_upgrade( + 'app', + from_series='trusty', + to_series='xenial', + ) + mock_async_set_series.assert_called_once_with( + 'app', to_series='xenial') + self.juju_status.assert_called() + # The below is using `any_order=True` because the ordering is + # undetermined and differs between python versions + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + mock.call('0', to_series='xenial'), + ], any_order=True) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '1', + origin='openstack-origin', + application='app', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '2', + origin='openstack-origin', + application='app', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '0', + origin='openstack-origin', + application='app', + files=None, + workaround_script=None, + post_upgrade_functions=None), + ]) + mock_post_application_upgrade_functions.assert_called_once_with(None) + + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_serial_series_upgrade( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + ): + await upgrade_utils.serial_series_upgrade( + 'app', + from_series='trusty', + to_series='xenial', + ) + mock_async_set_series.assert_called_once_with( + 'app', to_series='xenial') + self.juju_status.assert_called() + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('0', to_series='xenial'), + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + ]) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '0', + origin='openstack-origin', + application='app', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '1', + origin='openstack-origin', + application='app', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '2', + origin='openstack-origin', + application='app', + files=None, + workaround_script=None, + post_upgrade_functions=None), + ]) + mock_post_application_upgrade_functions.assert_called_once_with(None) + + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_complete_series_upgrade') + @mock.patch.object(upgrade_utils, 'reboot') + @mock.patch.object(upgrade_utils, 'async_do_release_upgrade') + @mock.patch.object(upgrade_utils, 'async_dist_upgrade') + async def test_series_upgrade_machine( + self, + mock_async_dist_upgrade, + mock_async_do_release_upgrade, + mock_reboot, + mock_async_complete_series_upgrade + ): + await upgrade_utils.series_upgrade_machine( + '1', + post_upgrade_functions=None, + pre_upgrade_functions=None, + files=None, + workaround_script=None) + mock_async_dist_upgrade.assert_called_once_with('1') + mock_async_do_release_upgrade.assert_called_once_with('1') + mock_reboot.assert_called_once_with('1') + mock_async_complete_series_upgrade.assert_called_once_with('1') + + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_complete_series_upgrade') + @mock.patch.object(upgrade_utils, 'reboot') + @mock.patch.object(upgrade_utils, 'async_do_release_upgrade') + @mock.patch.object(upgrade_utils, 'async_dist_upgrade') + async def test_series_upgrade_machine_with_source( + self, + mock_async_dist_upgrade, + mock_async_do_release_upgrade, + mock_reboot, + mock_async_complete_series_upgrade, + mock_async_set_origin + ): + await upgrade_utils.series_upgrade_machine( + '1', + origin='openstack-origin', + application='app', + post_upgrade_functions=None, + pre_upgrade_functions=None, + files=None, + workaround_script=None) + mock_async_dist_upgrade.assert_called_once_with('1') + mock_async_do_release_upgrade.assert_called_once_with('1') + mock_reboot.assert_called_once_with('1') + mock_async_complete_series_upgrade.assert_called_once_with('1') + mock_async_set_origin.assert_called_once_with( + 'app', 'openstack-origin') + + async def test_maybe_pause_things_primary(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=False, + pause_non_leader_primary=True) + self.async_run_action.assert_has_calls([ + mock.call('app/1', "pause", action_params={}), + mock.call('app/2', "pause", action_params={}), + ]) + + async def test_maybe_pause_things_subordinates(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=True, + pause_non_leader_primary=False) + self.async_run_action.assert_has_calls([ + mock.call('app-hacluster/1', "pause", action_params={}), + mock.call('app-hacluster/2', "pause", action_params={}), + ]) + + async def test_maybe_pause_things_all(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=True, + pause_non_leader_primary=True) + self.async_run_action.assert_has_calls([ + mock.call('app-hacluster/1', "pause", action_params={}), + mock.call('app/1', "pause", action_params={}), + mock.call('app-hacluster/2', "pause", action_params={}), + mock.call('app/2', "pause", action_params={}), + ]) + + async def test_maybe_pause_things_none(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=False, + pause_non_leader_primary=False) + self.async_run_action.assert_not_called() + + async def test_async_do_release_upgrade(self): + await upgrade_utils.async_do_release_upgrade('1') + do_release_upgrade_cmd = ( + 'yes | sudo DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive') + self.async_run_on_machine.assert_called_once_with( + '1', do_release_upgrade_cmd, timeout='120m' + ) + + async def test_prepare_series_upgrade(self): + await upgrade_utils.prepare_series_upgrade( + '1', to_series='xenial' + ) + self.async_prepare_series_upgrade.assert_called_once_with( + '1', to_series='xenial' + ) + + async def test_reboot(self): + await upgrade_utils.reboot('1') + self.async_run_on_machine.assert_called_once_with( + '1', 'sudo init 6 & exit' + ) + + async def test_async_dist_upgrade(self): + await upgrade_utils.async_dist_upgrade('1') + apt_update_command = ( + """yes | sudo DEBIAN_FRONTEND=noninteractive """ + """apt-get --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + self.async_run_on_machine.assert_has_calls([ + mock.call('1', 'sudo apt-get update'), + mock.call('1', apt_update_command), + ]) diff --git a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py index 1f21b49..814772e 100644 --- a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py +++ b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py @@ -90,6 +90,7 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): def test_get_upgrade_groups(self): expected = collections.OrderedDict([ + ('Stateful Services', []), ('Core Identity', []), ('Control Plane', ['cinder']), ('Data Plane', ['nova-compute']), @@ -103,10 +104,11 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): def test_get_series_upgrade_groups(self): expected = collections.OrderedDict([ + ('Stateful Services', ['mydb']), ('Core Identity', []), ('Control Plane', ['cinder']), ('Data Plane', ['nova-compute']), - ('sweep_up', ['mydb', 'ntp'])]) + ('sweep_up', ['ntp'])]) actual = openstack_upgrade.get_series_upgrade_groups() pprint.pprint(expected) pprint.pprint(actual) diff --git a/zaza/openstack/charm_tests/mysql/utils.py b/zaza/openstack/charm_tests/mysql/utils.py new file mode 100644 index 0000000..1fe5114 --- /dev/null +++ b/zaza/openstack/charm_tests/mysql/utils.py @@ -0,0 +1,26 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module of functions for interfacing with the percona-cluster charm.""" + +import zaza.model as model + + +async def complete_cluster_series_upgrade(): + """Run the complete-cluster-series-upgrade action on the lead unit.""" + # TODO: Make this work across either mysql or percona-cluster names + await model.async_run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 9826add..3994178 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -584,3 +584,11 @@ def _post_check_unit_cluster_nodes(unit, nodes, unit_node_names): 'after unit removal.\n' ''.format(unit_name, node)) return errors + + +async def complete_cluster_series_upgrade(): + """Run the complete-cluster-series-upgrade action on the lead unit.""" + await zaza.model.async_run_action_on_leader( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py new file mode 100644 index 0000000..6408cf0 --- /dev/null +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define class for Series Upgrade.""" + +import asyncio +import logging +import os +import sys +import unittest + +from zaza import model +from zaza.openstack.utilities import ( + cli as cli_utils, + upgrade_utils as upgrade_utils, +) +from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest +from zaza.openstack.utilities import ( + parallel_series_upgrade, +) + + +def _filter_easyrsa(app, app_config, model_name=None): + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "easyrsa" in charm_name: + logging.warn("Skipping series upgrade of easyrsa Bug #1850121") + return True + return False + + +def _filter_etcd(app, app_config, model_name=None): + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "etcd" in charm_name: + logging.warn("Skipping series upgrade of easyrsa Bug #1850124") + return True + return False + + +class ParallelSeriesUpgradeTest(unittest.TestCase): + """Class to encapsulate Series Upgrade Tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + cli_utils.setup_logging() + cls.from_series = None + cls.to_series = None + cls.workaround_script = None + cls.files = [] + + def test_200_run_series_upgrade(self): + """Run series upgrade.""" + # Set Feature Flag + os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + upgrade_groups = upgrade_utils.get_series_upgrade_groups( + extra_filters=[_filter_etcd, _filter_easyrsa]) + from_series = self.from_series + to_series = self.to_series + completed_machines = [] + workaround_script = None + files = [] + applications = model.get_status().applications + for group_name, apps in upgrade_groups.items(): + logging.info("About to upgrade {} from {} to {}".format( + group_name, from_series, to_series)) + upgrade_functions = [] + if group_name in ["Stateful Services", "Data Plane", "sweep_up"]: + logging.info("Going to upgrade {} unit by unit".format(apps)) + upgrade_function = \ + parallel_series_upgrade.serial_series_upgrade + else: + logging.info("Going to upgrade {} all at once".format(apps)) + upgrade_function = \ + parallel_series_upgrade.parallel_series_upgrade + + for charm_name in apps: + charm = applications[charm_name]['charm'] + name = upgrade_utils.extract_charm_name_from_url(charm) + upgrade_config = parallel_series_upgrade.app_config(name) + upgrade_functions.append( + upgrade_function( + charm_name, + **upgrade_config, + from_series=from_series, + to_series=to_series, + completed_machines=completed_machines, + workaround_script=workaround_script, + files=files)) + asyncio.get_event_loop().run_until_complete( + asyncio.gather(*upgrade_functions)) + model.block_until_all_units_idle() + logging.info("Finished {}".format(group_name)) + logging.info("Done!") + + +class OpenStackParallelSeriesUpgrade(ParallelSeriesUpgradeTest): + """OpenStack Series Upgrade. + + Full OpenStack series upgrade with VM launch before and after the series + upgrade. + + This test requires a full OpenStack including at least: keystone, glance, + nova-cloud-controller, nova-compute, neutron-gateway, neutron-api and + neutron-openvswitch. + """ + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + super(OpenStackParallelSeriesUpgrade, cls).setUpClass() + cls.lts = LTSGuestCreateTest() + cls.lts.setUpClass() + + def test_100_validate_pre_series_upgrade_cloud(self): + """Validate pre series upgrade.""" + logging.info("Validate pre-series-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + def test_500_validate_series_upgraded_cloud(self): + """Validate post series upgrade.""" + logging.info("Validate post-series-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + +class TrustyXenialSeriesUpgrade(OpenStackParallelSeriesUpgrade): + """OpenStack Trusty to Xenial Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Trusty to Xenial Series Upgrades.""" + super(TrustyXenialSeriesUpgrade, cls).setUpClass() + cls.from_series = "trusty" + cls.to_series = "xenial" + + +class XenialBionicSeriesUpgrade(OpenStackParallelSeriesUpgrade): + """OpenStack Xenial to Bionic Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(XenialBionicSeriesUpgrade, cls).setUpClass() + cls.from_series = "xenial" + cls.to_series = "bionic" + + +class BionicFocalSeriesUpgrade(OpenStackParallelSeriesUpgrade): + """OpenStack Bionic to FocalSeries Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(BionicFocalSeriesUpgrade, cls).setUpClass() + cls.from_series = "bionic" + cls.to_series = "focal" + + +class UbuntuLiteParallelSeriesUpgrade(unittest.TestCase): + """ubuntu Lite Parallel Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + cli_utils.setup_logging() + cls.from_series = None + cls.to_series = None + + def test_200_run_series_upgrade(self): + """Run series upgrade.""" + # Set Feature Flag + os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + parallel_series_upgrade.upgrade_ubuntu_lite( + from_series=self.from_series, + to_series=self.to_series + ) + + +class TrustyXenialSeriesUpgradeUbuntu(UbuntuLiteParallelSeriesUpgrade): + """OpenStack Trusty to Xenial Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Trusty to Xenial Series Upgrades.""" + super(TrustyXenialSeriesUpgradeUbuntu, cls).setUpClass() + cls.from_series = "trusty" + cls.to_series = "xenial" + + +class XenialBionicSeriesUpgradeUbuntu(UbuntuLiteParallelSeriesUpgrade): + """OpenStack Xenial to Bionic Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(XenialBionicSeriesUpgradeUbuntu, cls).setUpClass() + cls.from_series = "xenial" + cls.to_series = "bionic" + + +class BionicFocalSeriesUpgradeUbuntu(UbuntuLiteParallelSeriesUpgrade): + """OpenStack Bionic to FocalSeries Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(BionicFocalSeriesUpgradeUbuntu, cls).setUpClass() + cls.from_series = "bionic" + cls.to_series = "focal" + + +if __name__ == "__main__": + from_series = os.environ.get("FROM_SERIES") + if from_series == "trusty": + to_series = "xenial" + series_upgrade_test = TrustyXenialSeriesUpgrade() + elif from_series == "xenial": + to_series = "bionic" + series_upgrade_test = XenialBionicSeriesUpgrade() + elif from_series == "bionic": + to_series = "focal" + series_upgrade_test = BionicFocalSeriesUpgrade() + + else: + raise Exception("FROM_SERIES is not set to a vailid LTS series") + series_upgrade_test.setUpClass() + sys.exit(series_upgrade_test.test_200_run_series_upgrade()) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 96c45cb..21c3793 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -66,6 +66,20 @@ def mojo_unseal_by_unit(): zaza.model.run_on_unit(unit_name, './hooks/update-status') +async def async_mojo_unseal_by_unit(): + """Unseal any units reported as sealed using mojo cacert.""" + cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path() + vault_creds = vault_utils.get_credentails() + for client in vault_utils.get_clients(cacert=cacert): + if client.hvac_client.is_sealed(): + client.hvac_client.unseal(vault_creds['keys'][0]) + unit_name = await juju_utils.async_get_unit_name_from_ip_address( + client.addr, + 'vault') + await zaza.model.async_run_on_unit( + unit_name, './hooks/update-status') + + def auto_initialize(cacert=None, validation_application='keystone'): """Auto initialize vault for testing. diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 50dcc69..0874cc6 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -385,10 +385,17 @@ async def check_call(cmd): stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) stdout, stderr = await proc.communicate() + stdout = stdout.decode('utf-8') + stderr = stderr.decode('utf-8') if proc.returncode != 0: logging.warn("STDOUT: {}".format(stdout)) logging.warn("STDERR: {}".format(stderr)) raise subprocess.CalledProcessError(proc.returncode, cmd) + else: + if stderr: + logging.info("STDERR: {} ({})".format(stderr, ' '.join(cmd))) + if stdout: + logging.info("STDOUT: {} ({})".format(stdout, ' '.join(cmd))) def set_dpkg_non_interactive_on_unit( diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py new file mode 100755 index 0000000..29e276d --- /dev/null +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -0,0 +1,573 @@ +#!/usr/bin/env python3 +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of functions for testing series upgrade in parallel.""" + + +import asyncio + +import collections +import copy +import logging +import subprocess + +from zaza import model +from zaza.charm_lifecycle import utils as cl_utils +import zaza.openstack.utilities.generic as os_utils +import zaza.openstack.utilities.series_upgrade as series_upgrade_utils +from zaza.openstack.utilities.series_upgrade import ( + SUBORDINATE_PAUSE_RESUME_BLACKLIST, +) + + +def app_config(charm_name): + """Return a dict with the upgrade config for an application. + + :param charm_name: Name of the charm about to upgrade + :type charm_name: str + :param async: Whether the upgreade functions should be async + :type async: bool + :returns: A dicitonary of the upgrade config for the application + :rtype: Dict + """ + default = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': False, } + _app_settings = collections.defaultdict(lambda: default) + ceph = { + 'origin': "source", + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + } + exceptions = { + 'rabbitmq-server': { + 'origin': 'source', + 'pause_non_leader_subordinate': False, + 'post_application_upgrade_functions': [ + ('zaza.openstack.charm_tests.rabbitmq_server.utils.' + 'complete_cluster_series_upgrade')] + }, + 'percona-cluster': { + 'origin': 'source', + 'post_application_upgrade_functions': [ + ('zaza.openstack.charm_tests.mysql.utils.' + 'complete_cluster_series_upgrade')] + }, + 'nova-compute': { + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + # TODO + # 'pre_upgrade_functions': [ + # 'zaza.openstack.charm_tests.nova_compute.setup.evacuate' + # ] + }, + 'ceph': ceph, + 'ceph-mon': ceph, + 'ceph-osd': ceph, + 'designate-bind': {'origin': None, }, + 'tempest': {'origin': None, }, + 'memcached': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + }, + 'vault': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': True, + 'post_upgrade_functions': [ + ('zaza.openstack.charm_tests.vault.setup.' + 'async_mojo_unseal_by_unit')] + }, + 'mongodb': { + 'origin': None, + 'follower_first': True, + } + } + for key, value in exceptions.items(): + _app_settings[key] = copy.deepcopy(default) + _app_settings[key].update(value) + return _app_settings[charm_name] + + +def upgrade_ubuntu_lite(from_series='xenial', to_series='bionic'): + """Validate that we can upgrade the ubuntu-lite charm. + + :param from_series: What series are we upgrading from + :type from_series: str + :param to_series: What series are we upgrading to + :type to_series: str + """ + completed_machines = [] + asyncio.get_event_loop().run_until_complete( + parallel_series_upgrade( + 'ubuntu-lite', pause_non_leader_primary=False, + pause_non_leader_subordinate=False, + completed_machines=completed_machines, origin=None) + ) + + +async def parallel_series_upgrade( + application, + from_series='xenial', + to_series='bionic', + origin='openstack-origin', + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + pre_upgrade_functions=None, + post_upgrade_functions=None, + post_application_upgrade_functions=None, + completed_machines=None, + follower_first=False, + files=None, + workaround_script=None +): + """Perform series upgrade on an application in parallel. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + + :param pre_upgrade_functions: A list of Zaza functions to call before + the upgrade is started on each machine + :type pre_upgrade_functions: List[str] + :param post_upgrade_functions: A list of Zaza functions to call when + the upgrade is complete on each machine + :type post_upgrade_functions: List[str] + :param post_application_upgrade_functions: A list of Zaza functions + to call when the upgrade is complete + on all machine in the application + :param follower_first: Should the follower(s) be upgraded first + :type follower_first: bool + :type post_application_upgrade_functions: List[str] + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + if completed_machines is None: + completed_machines = [] + if follower_first: + logging.error("leader_first is ignored for parallel upgrade") + logging.info( + "About to upgrade the units of {} in parallel (follower first: {})" + .format(application, follower_first)) + + status = (await model.async_get_status()).applications[application] + logging.info( + "Configuring leader / non leaders for {}".format(application)) + leader, non_leaders = get_leader_and_non_leaders(status) + for leader_name, leader_unit in leader.items(): + leader_machine = leader_unit["machine"] + leader = leader_name + machines = [ + unit["machine"] for name, unit + in non_leaders.items() + if unit['machine'] not in completed_machines] + + await maybe_pause_things( + status, + non_leaders, + pause_non_leader_subordinate, + pause_non_leader_primary) + await series_upgrade_utils.async_set_series( + application, to_series=to_series) + app_idle = [ + model.async_wait_for_unit_idle(unit) for unit in status["units"] + ] + await asyncio.gather(*app_idle) + prepare_group = [ + prepare_series_upgrade(machine, to_series=to_series) + for machine in machines] + await asyncio.gather(*prepare_group) + await prepare_series_upgrade(leader_machine, to_series=to_series) + if leader_machine not in completed_machines: + machines.append(leader_machine) + upgrade_group = [ + series_upgrade_machine( + machine, + origin=origin, + application=application, + files=files, workaround_script=workaround_script, + post_upgrade_functions=post_upgrade_functions) + for machine in machines + ] + await asyncio.gather(*upgrade_group) + completed_machines.extend(machines) + await run_post_application_upgrade_functions( + post_application_upgrade_functions) + + +async def serial_series_upgrade( + application, + from_series='xenial', + to_series='bionic', + origin='openstack-origin', + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + pre_upgrade_functions=None, + post_upgrade_functions=None, + post_application_upgrade_functions=None, + completed_machines=None, + follower_first=False, + files=None, + workaround_script=None +): + """Perform series upgrade on an application in serial. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + + :param pre_upgrade_functions: A list of Zaza functions to call before + the upgrade is started on each machine + :type pre_upgrade_functions: List[str] + :param post_upgrade_functions: A list of Zaza functions to call when + the upgrade is complete on each machine + :type post_upgrade_functions: List[str] + :param post_application_upgrade_functions: A list of Zaza functions + to call when the upgrade is complete + on all machine in the application + :param follower_first: Should the follower(s) be upgraded first + :type follower_first: bool + :type post_application_upgrade_functions: List[str] + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + if completed_machines is None: + completed_machines = [] + logging.info( + "About to upgrade the units of {} in serial (follower first: {})" + .format(application, follower_first)) + status = (await model.async_get_status()).applications[application] + logging.info( + "Configuring leader / non leaders for {}".format(application)) + leader, non_leaders = get_leader_and_non_leaders(status) + for leader_name, leader_unit in leader.items(): + leader_machine = leader_unit["machine"] + leader = leader_name + + await maybe_pause_things( + status, + non_leaders, + pause_non_leader_subordinate, + pause_non_leader_primary) + await series_upgrade_utils.async_set_series( + application, to_series=to_series) + if not follower_first and leader_machine not in completed_machines: + await model.async_wait_for_unit_idle(leader) + await prepare_series_upgrade(leader_machine, to_series=to_series) + logging.info("About to upgrade leader of {}: {}" + .format(application, leader_machine)) + await series_upgrade_machine( + leader_machine, + origin=origin, + application=application, + files=files, workaround_script=workaround_script, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(leader_machine) + + # for machine in machines: + for unit_name, unit in non_leaders.items(): + machine = unit['machine'] + if machine in completed_machines: + continue + await model.async_wait_for_unit_idle(unit_name) + await prepare_series_upgrade(machine, to_series=to_series) + logging.info("About to upgrade follower of {}: {}" + .format(application, machine)) + await series_upgrade_machine( + machine, + origin=origin, + application=application, + files=files, workaround_script=workaround_script, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + + if follower_first and leader_machine not in completed_machines: + await model.async_wait_for_unit_idle(leader) + await prepare_series_upgrade(leader_machine, to_series=to_series) + logging.info("About to upgrade leader of {}: {}" + .format(application, leader_machine)) + await series_upgrade_machine( + leader_machine, + origin=origin, + application=application, + files=files, workaround_script=workaround_script, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(leader_machine) + await run_post_application_upgrade_functions( + post_application_upgrade_functions) + + +async def series_upgrade_machine( + machine, + origin=None, + application=None, + post_upgrade_functions=None, + pre_upgrade_functions=None, + files=None, + workaround_script=None): + """Perform series upgrade on an machine. + + :param machine_num: Machine number + :type machine_num: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :param pre_upgrade_functions: A list of Zaza functions to call before + the upgrade is started on each machine + :type pre_upgrade_functions: List[str] + :param post_upgrade_functions: A list of Zaza functions to call when + the upgrade is complete on each machine + :type post_upgrade_functions: List[str] + :returns: None + :rtype: None + """ + logging.info( + "About to series-upgrade ({})".format(machine)) + await run_pre_upgrade_functions(machine, pre_upgrade_functions) + await async_dist_upgrade(machine) + await async_do_release_upgrade(machine) + await reboot(machine) + if origin: + await os_utils.async_set_origin(application, origin) + await series_upgrade_utils.async_complete_series_upgrade(machine) + await run_post_upgrade_functions(post_upgrade_functions) + + +async def run_pre_upgrade_functions(machine, pre_upgrade_functions): + """Execute list supplied functions. + + Each of the supplied functions will be called with a single + argument of the machine that is about to be upgraded. + + :param machine: Machine that is about to be upgraded + :type machine: str + :param pre_upgrade_functions: List of awaitable functions + :type pre_upgrade_functions: [function, function, ...] + """ + if pre_upgrade_functions: + for func in pre_upgrade_functions: + logging.info("Running {}".format(func)) + m = cl_utils.get_class(func) + await m(machine) + + +async def run_post_upgrade_functions(post_upgrade_functions): + """Execute list supplied functions. + + :param post_upgrade_functions: List of awaitable functions + :type post_upgrade_functions: [function, function, ...] + """ + if post_upgrade_functions: + for func in post_upgrade_functions: + logging.info("Running {}".format(func)) + m = cl_utils.get_class(func) + await m() + + +async def run_post_application_upgrade_functions(post_upgrade_functions): + """Execute list supplied functions. + + :param post_upgrade_functions: List of awaitable functions + :type post_upgrade_functions: [function, function, ...] + """ + if post_upgrade_functions: + for func in post_upgrade_functions: + logging.info("Running {}".format(func)) + m = cl_utils.get_class(func) + await m() + + +async def maybe_pause_things( + status, units, pause_non_leader_subordinate=True, + pause_non_leader_primary=True): + """Pause the non-leaders, based on the run configuration. + + :param status: Juju status for an application + :type status: juju.applications + :param units: List of units to paybe pause + :type units: LIst[str] + :param pause_non_leader_subordinate: Should the non leader + subordinate be paused + :type pause_non_leader_subordinate: bool + :param pause_non_leader_primary: Should the non leaders be paused + :type pause_non_leader_primary: bool + :returns: Nothing + :trype: None + """ + subordinate_pauses = [] + leader_pauses = [] + for unit in units: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + _app = subordinate.split('/')[0] + if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: + logging.info("Skipping pausing {} - blacklisted" + .format(subordinate)) + else: + logging.info("Pausing {}".format(subordinate)) + subordinate_pauses.append(model.async_run_action( + subordinate, "pause", action_params={})) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + leader_pauses.append( + model.async_run_action(unit, "pause", action_params={})) + await asyncio.gather(*leader_pauses) + await asyncio.gather(*subordinate_pauses) + + +def get_leader_and_non_leaders(status): + """Get the leader and non-leader Juju units. + + This function returns a tuple that looks like: + + ({ + 'unit/1': juju.Unit, + }, + { + 'unit/0': juju.Unit, + 'unit/2': juju.unit, + }) + + The first entry of this tuple is the leader, and the second is + all non-leader units. + + :returns: A tuple of dicts identifying leader and non-leaders + :rtype: Dict[str, List[juju.Unit]] + """ + leader = None + non_leaders = {} + for name, unit in status["units"].items(): + if unit.get("leader"): + leader = {name: unit} + else: + non_leaders[name] = unit + return (leader, non_leaders) + + +async def prepare_series_upgrade(machine, to_series): + """Execute juju series-upgrade prepare on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + :param machine_num: Machine number + :type machine_num: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + logging.info("Preparing series upgrade for: {}".format(machine)) + await series_upgrade_utils.async_prepare_series_upgrade( + machine, to_series=to_series) + + +async def reboot(machine): + """Reboot the named machine. + + :param machine: Machine to reboot + :type machine: str + :returns: Nothing + :rtype: None + """ + try: + await model.async_run_on_machine(machine, 'sudo init 6 & exit') + # await run_on_machine(unit, "sudo reboot && exit") + except subprocess.CalledProcessError as e: + logging.warn("Error doing reboot: {}".format(e)) + pass + + +async def async_dist_upgrade(machine): + """Run dist-upgrade on unit after update package db. + + :param machine: Machine Number + :type machine: str + :returns: None + :rtype: None + """ + logging.info('Updating package db ' + machine) + update_cmd = 'sudo apt-get update' + await model.async_run_on_machine(machine, update_cmd) + + logging.info('Updating existing packages ' + machine) + dist_upgrade_cmd = ( + """yes | sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + await model.async_run_on_machine(machine, dist_upgrade_cmd) + + +async def async_do_release_upgrade(machine): + """Run do-release-upgrade noninteractive. + + :param machine: Machine Name + :type machine: str + :returns: None + :rtype: None + """ + logging.info('Upgrading ' + machine) + do_release_upgrade_cmd = ( + 'yes | sudo DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive') + + await model.async_run_on_machine( + machine, do_release_upgrade_cmd, timeout="120m") diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 7f3efa3..f7e79a8 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -687,6 +687,7 @@ async def async_complete_series_upgrade(machine_num): juju_model = await model.async_get_juju_model() cmd = ["juju", "upgrade-series", "-m", juju_model, machine_num, "complete"] + logging.info("About to call '{}'".format(cmd)) await os_utils.check_call(cmd) @@ -704,6 +705,7 @@ async def async_set_series(application, to_series): juju_model = await model.async_get_juju_model() cmd = ["juju", "set-series", "-m", juju_model, application, to_series] + logging.info("About to call '{}'".format(cmd)) await os_utils.check_call(cmd) diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 062e9b1..4066d19 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -20,15 +20,17 @@ import zaza.model SERVICE_GROUPS = collections.OrderedDict([ + ('Stateful Services', ['percona-cluster', 'rabbitmq-server', 'ceph-mon']), ('Core Identity', ['keystone']), ('Control Plane', [ - 'aodh', 'barbican', 'ceilometer', 'ceph-mon', 'ceph-fs', + 'aodh', 'barbican', 'ceilometer', 'ceph-fs', 'ceph-radosgw', 'cinder', 'designate', 'designate-bind', 'glance', 'gnocchi', 'heat', 'manila', 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', 'nova-cloud-controller', 'openstack-dashboard']), ('Data Plane', [ - 'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage']) + 'nova-compute', 'ceph-osd', + 'swift-proxy', 'swift-storage']) ]) UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster']