diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index d18b646..0f3ff9c 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -173,45 +173,6 @@ class TestGenericUtils(ut_utils.BaseTestCase): _yaml_dict) self._open.assert_called_once_with(_filename, "r") - def test_dist_upgrade(self): - _unit = "app/2" - generic_utils.dist_upgrade(_unit) - dist_upgrade_cmd = ( - """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ - """-o "Dpkg::Options::=--force-confdef" """ - """-o "Dpkg::Options::=--force-confold" dist-upgrade""") - self.model.run_on_unit.assert_has_calls([ - mock.call(_unit, 'sudo apt update'), - mock.call(_unit, dist_upgrade_cmd)]) - - def test_do_release_upgrade(self): - _unit = "app/2" - generic_utils.do_release_upgrade(_unit) - self.subprocess.check_call.assert_called_once_with( - ['juju', 'ssh', _unit, 'sudo', 'DEBIAN_FRONTEND=noninteractive', - 'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive']) - - def test_wrap_do_release_upgrade(self): - self.patch_object(generic_utils, "do_release_upgrade") - self.patch_object(generic_utils, "run_via_ssh") - self.patch_object(generic_utils.model, "scp_to_unit") - _unit = "app/2" - _from_series = "xenial" - _to_series = "bionic" - _workaround_script = "scriptname" - _files = ["filename", _workaround_script] - _scp_calls = [] - _run_calls = [ - mock.call(_unit, _workaround_script)] - for filename in _files: - _scp_calls.append(mock.call(_unit, filename, filename)) - generic_utils.wrap_do_release_upgrade( - _unit, to_series=_to_series, from_series=_from_series, - workaround_script=_workaround_script, files=_files) - self.scp_to_unit.assert_has_calls(_scp_calls) - self.run_via_ssh.assert_has_calls(_run_calls) - self.do_release_upgrade.assert_called_once_with(_unit) - def test_reboot(self): _unit = "app/2" generic_utils.reboot(_unit) @@ -237,149 +198,6 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.set_application_config.assert_called_once_with( _application, {_origin: _pocket}) - def test_series_upgrade(self): - self.patch_object(generic_utils.model, "block_until_all_units_idle") - self.patch_object(generic_utils.model, "block_until_unit_wl_status") - self.patch_object(generic_utils.model, "prepare_series_upgrade") - self.patch_object(generic_utils.model, "complete_series_upgrade") - self.patch_object(generic_utils.model, "set_series") - self.patch_object(generic_utils, "set_origin") - self.patch_object(generic_utils, "wrap_do_release_upgrade") - self.patch_object(generic_utils, "reboot") - _unit = "app/2" - _application = "app" - _machine_num = "4" - _from_series = "xenial" - _to_series = "bionic" - _origin = "source" - _files = ["filename", "scriptname"] - _workaround_script = "scriptname" - generic_utils.series_upgrade( - _unit, _machine_num, origin=_origin, - to_series=_to_series, from_series=_from_series, - workaround_script=_workaround_script, files=_files) - self.block_until_all_units_idle.called_with() - self.prepare_series_upgrade.assert_called_once_with( - _machine_num, to_series=_to_series) - self.wrap_do_release_upgrade.assert_called_once_with( - _unit, to_series=_to_series, from_series=_from_series, - workaround_script=_workaround_script, files=_files) - self.complete_series_upgrade.assert_called_once_with(_machine_num) - self.set_series.assert_called_once_with(_application, _to_series) - self.set_origin.assert_called_once_with(_application, _origin) - self.reboot.assert_called_once_with(_unit) - - def test_series_upgrade_application_pause_peers_and_subordinates(self): - self.patch_object(generic_utils.model, "run_action") - self.patch_object(generic_utils, "series_upgrade") - _application = "app" - _from_series = "xenial" - _to_series = "bionic" - _origin = "source" - _files = ["filename", "scriptname"] - _workaround_script = "scriptname" - _completed_machines = [] - # Peers and Subordinates - _run_action_calls = [ - mock.call("{}-hacluster/1".format(_application), - "pause", action_params={}), - mock.call("{}/1".format(_application), "pause", action_params={}), - mock.call("{}-hacluster/2".format(_application), - "pause", action_params={}), - mock.call("{}/2".format(_application), "pause", action_params={}), - ] - _series_upgrade_calls = [] - for machine_num in ("0", "1", "2"): - _series_upgrade_calls.append( - mock.call("{}/{}".format(_application, machine_num), - machine_num, origin=_origin, - from_series=_from_series, to_series=_to_series, - workaround_script=_workaround_script, files=_files, - post_upgrade_functions=None), - ) - - # Pause primary peers and subordinates - generic_utils.series_upgrade_application( - _application, origin=_origin, - to_series=_to_series, from_series=_from_series, - pause_non_leader_primary=True, - pause_non_leader_subordinate=True, - completed_machines=_completed_machines, - workaround_script=_workaround_script, files=_files), - self.run_action.assert_has_calls(_run_action_calls) - self.series_upgrade.assert_has_calls(_series_upgrade_calls) - - def test_series_upgrade_application_pause_subordinates(self): - self.patch_object(generic_utils.model, "run_action") - self.patch_object(generic_utils, "series_upgrade") - _application = "app" - _from_series = "xenial" - _to_series = "bionic" - _origin = "source" - _files = ["filename", "scriptname"] - _workaround_script = "scriptname" - _completed_machines = [] - # Subordinates only - _run_action_calls = [ - mock.call("{}-hacluster/1".format(_application), - "pause", action_params={}), - mock.call("{}-hacluster/2".format(_application), - "pause", action_params={}), - ] - _series_upgrade_calls = [] - - for machine_num in ("0", "1", "2"): - _series_upgrade_calls.append( - mock.call("{}/{}".format(_application, machine_num), - machine_num, origin=_origin, - from_series=_from_series, to_series=_to_series, - workaround_script=_workaround_script, files=_files, - post_upgrade_functions=None), - ) - - # Pause subordinates - generic_utils.series_upgrade_application( - _application, origin=_origin, - to_series=_to_series, from_series=_from_series, - pause_non_leader_primary=False, - pause_non_leader_subordinate=True, - completed_machines=_completed_machines, - workaround_script=_workaround_script, files=_files), - self.run_action.assert_has_calls(_run_action_calls) - self.series_upgrade.assert_has_calls(_series_upgrade_calls) - - def test_series_upgrade_application_no_pause(self): - self.patch_object(generic_utils.model, "run_action") - self.patch_object(generic_utils, "series_upgrade") - _application = "app" - _from_series = "xenial" - _to_series = "bionic" - _origin = "source" - _series_upgrade_calls = [] - _files = ["filename", "scriptname"] - _workaround_script = "scriptname" - _completed_machines = [] - - for machine_num in ("0", "1", "2"): - _series_upgrade_calls.append( - mock.call("{}/{}".format(_application, machine_num), - machine_num, origin=_origin, - from_series=_from_series, to_series=_to_series, - workaround_script=_workaround_script, files=_files, - post_upgrade_functions=None), - ) - - # No Pausiing - generic_utils.series_upgrade_application( - _application, origin=_origin, - to_series=_to_series, from_series=_from_series, - pause_non_leader_primary=False, - pause_non_leader_subordinate=False, - completed_machines=_completed_machines, - workaround_script=_workaround_script, files=_files) - self.run_action.assert_not_called() - self.series_upgrade.assert_has_calls(_series_upgrade_calls) - def test_set_dpkg_non_interactive_on_unit(self): self.patch_object(generic_utils, "model") _unit_name = "app/1" diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py index f711bb2..4d619c5 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy import mock import unit_tests.utils as ut_utils @@ -172,38 +171,6 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): {'source': 'new-src'}, model_name=None) - def test__extract_charm_name_from_url(self): - self.assertEqual( - openstack_upgrade._extract_charm_name_from_url( - 'local:bionic/heat-12'), - 'heat') - self.assertEqual( - openstack_upgrade._extract_charm_name_from_url( - 'cs:bionic/heat-12'), - 'heat') - self.assertEqual( - openstack_upgrade._extract_charm_name_from_url('cs:heat'), - 'heat') - - def test_get_upgrade_candidates(self): - expect = copy.deepcopy(self.juju_status.applications) - del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST - del expect['ntp'] # Filter as it has no source option - del expect['neutron-openvswitch'] # Filter as it is a subordinates - self.assertEqual( - openstack_upgrade.get_upgrade_candidates(), - expect) - - def test_get_upgrade_groups(self): - self.assertEqual( - openstack_upgrade.get_upgrade_groups(), - { - 'Compute': ['nova-compute'], - 'Control Plane': ['cinder'], - 'Core Identity': [], - 'Storage': [], - 'sweep_up': []}) - def test_is_action_upgradable(self): self.assertTrue( openstack_upgrade.is_action_upgradable('cinder')) diff --git a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py new file mode 100644 index 0000000..b6f21e9 --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py @@ -0,0 +1,239 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unit_tests.utils as ut_utils +import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.series_upgrade as series_upgrade_utils + +FAKE_STATUS = { + 'can-upgrade-to': '', + 'charm': 'local:trusty/app-136', + 'subordinate-to': [], + 'units': {'app/0': {'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}, + 'app/1': {'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': {'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}} + + +class TestSeriesUpgrade(ut_utils.BaseTestCase): + def setUp(self): + super(TestSeriesUpgrade, self).setUp() + # Patch all subprocess calls + self.patch( + 'zaza.openstack.utilities.generic.subprocess', + new_callable=mock.MagicMock(), + name='subprocess' + ) + self.patch_object(generic_utils, "run_via_ssh") + # Juju Status Object and data + self.juju_status = mock.MagicMock() + self.juju_status.applications.__getitem__.return_value = FAKE_STATUS + self.patch_object(series_upgrade_utils, "model") + self.model.get_status.return_value = self.juju_status + + def test_series_upgrade(self): + self.patch_object( + series_upgrade_utils.model, "block_until_all_units_idle") + self.patch_object( + series_upgrade_utils.model, "block_until_unit_wl_status") + self.patch_object(series_upgrade_utils.model, "prepare_series_upgrade") + self.patch_object( + series_upgrade_utils.model, "complete_series_upgrade") + self.patch_object(series_upgrade_utils.model, "set_series") + self.patch_object(generic_utils, "set_origin") + self.patch_object(series_upgrade_utils, "wrap_do_release_upgrade") + self.patch_object(generic_utils, "reboot") + _unit = "app/2" + _application = "app" + _machine_num = "4" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + series_upgrade_utils.series_upgrade( + _unit, _machine_num, origin=_origin, + to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) + self.block_until_all_units_idle.called_with() + self.prepare_series_upgrade.assert_called_once_with( + _machine_num, to_series=_to_series) + self.wrap_do_release_upgrade.assert_called_once_with( + _unit, to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) + self.complete_series_upgrade.assert_called_once_with(_machine_num) + self.set_series.assert_called_once_with(_application, _to_series) + self.set_origin.assert_called_once_with(_application, _origin) + self.reboot.assert_called_once_with(_unit) + + def test_series_upgrade_application_pause_peers_and_subordinates(self): + self.patch_object(series_upgrade_utils.model, "run_action") + self.patch_object(series_upgrade_utils, "series_upgrade") + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + _completed_machines = [] + # Peers and Subordinates + _run_action_calls = [ + mock.call("{}-hacluster/1".format(_application), + "pause", action_params={}), + mock.call("{}/1".format(_application), "pause", action_params={}), + mock.call("{}-hacluster/2".format(_application), + "pause", action_params={}), + mock.call("{}/2".format(_application), "pause", action_params={}), + ] + _series_upgrade_calls = [] + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files, + post_upgrade_functions=None), + ) + + # Pause primary peers and subordinates + series_upgrade_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + completed_machines=_completed_machines, + workaround_script=_workaround_script, files=_files), + self.run_action.assert_has_calls(_run_action_calls) + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_series_upgrade_application_pause_subordinates(self): + self.patch_object(series_upgrade_utils.model, "run_action") + self.patch_object(series_upgrade_utils, "series_upgrade") + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + _completed_machines = [] + # Subordinates only + _run_action_calls = [ + mock.call("{}-hacluster/1".format(_application), + "pause", action_params={}), + mock.call("{}-hacluster/2".format(_application), + "pause", action_params={}), + ] + _series_upgrade_calls = [] + + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files, + post_upgrade_functions=None), + ) + + # Pause subordinates + series_upgrade_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=False, + pause_non_leader_subordinate=True, + completed_machines=_completed_machines, + workaround_script=_workaround_script, files=_files), + self.run_action.assert_has_calls(_run_action_calls) + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_series_upgrade_application_no_pause(self): + self.patch_object(series_upgrade_utils.model, "run_action") + self.patch_object(series_upgrade_utils, "series_upgrade") + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _series_upgrade_calls = [] + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + _completed_machines = [] + + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files, + post_upgrade_functions=None), + ) + + # No Pausiing + series_upgrade_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=False, + pause_non_leader_subordinate=False, + completed_machines=_completed_machines, + workaround_script=_workaround_script, files=_files) + self.run_action.assert_not_called() + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_dist_upgrade(self): + _unit = "app/2" + series_upgrade_utils.dist_upgrade(_unit) + dist_upgrade_cmd = ( + """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + self.model.run_on_unit.assert_has_calls([ + mock.call(_unit, 'sudo apt update'), + mock.call(_unit, dist_upgrade_cmd)]) + + def test_do_release_upgrade(self): + _unit = "app/2" + series_upgrade_utils.do_release_upgrade(_unit) + self.run_via_ssh.assert_called_once_with( + _unit, + 'DEBIAN_FRONTEND=noninteractive do-release-upgrade ' + '-f DistUpgradeViewNonInteractive') + + def test_wrap_do_release_upgrade(self): + self.patch_object(series_upgrade_utils, "do_release_upgrade") + self.patch_object(series_upgrade_utils.model, "scp_to_unit") + _unit = "app/2" + _from_series = "xenial" + _to_series = "bionic" + _workaround_script = "scriptname" + _files = ["filename", _workaround_script] + _scp_calls = [] + _run_calls = [ + mock.call(_unit, _workaround_script)] + for filename in _files: + _scp_calls.append(mock.call(_unit, filename, filename)) + series_upgrade_utils.wrap_do_release_upgrade( + _unit, to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) + self.scp_to_unit.assert_has_calls(_scp_calls) + self.run_via_ssh.assert_has_calls(_run_calls) + self.do_release_upgrade.assert_called_once_with(_unit) diff --git a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py new file mode 100644 index 0000000..1f21b49 --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py @@ -0,0 +1,128 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import copy +import mock +import pprint + +import unit_tests.utils as ut_utils +import zaza.openstack.utilities.upgrade_utils as openstack_upgrade + + +class TestUpgradeUtils(ut_utils.BaseTestCase): + def setUp(self): + super(TestUpgradeUtils, self).setUp() + self.patch_object( + openstack_upgrade.zaza.model, + "get_units") + self.juju_status = mock.MagicMock() + self.patch_object( + openstack_upgrade.zaza.model, + "get_status", + return_value=self.juju_status) + self.patch_object( + openstack_upgrade.zaza.model, + "get_application_config") + + def _get_application_config(app, model_name=None): + app_config = { + 'ceph-mon': {'verbose': True, 'source': 'old-src'}, + 'neutron-openvswitch': {'verbose': True}, + 'ntp': {'verbose': True}, + 'percona-cluster': {'verbose': True, 'source': 'old-src'}, + 'cinder': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + 'neutron-api': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + 'nova-compute': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + } + return app_config[app] + self.get_application_config.side_effect = _get_application_config + self.juju_status.applications = { + 'mydb': { # Filter as it is on UPGRADE_EXCLUDE_LIST + 'charm': 'cs:percona-cluster'}, + 'neutron-openvswitch': { # Filter as it is a subordinates + 'charm': 'cs:neutron-openvswitch', + 'subordinate-to': 'nova-compute'}, + 'ntp': { # Filter as it has no source option + 'charm': 'cs:ntp'}, + 'nova-compute': { + 'charm': 'cs:nova-compute', + 'units': { + 'nova-compute/0': { + 'subordinates': { + 'neutron-openvswitch/2': { + 'charm': 'cs:neutron-openvswitch-22'}}}}}, + 'cinder': { + 'charm': 'cs:cinder-23', + 'units': { + 'cinder/1': { + 'subordinates': { + 'cinder-hacluster/0': { + 'charm': 'cs:hacluster-42'}, + 'cinder-ceph/3': { + 'charm': 'cs:cinder-ceph-2'}}}}}} + + def test_get_upgrade_candidates(self): + expected = copy.deepcopy(self.juju_status.applications) + self.assertEqual( + openstack_upgrade.get_upgrade_candidates(), + expected) + + def test_get_upgrade_groups(self): + expected = collections.OrderedDict([ + ('Core Identity', []), + ('Control Plane', ['cinder']), + ('Data Plane', ['nova-compute']), + ('sweep_up', [])]) + actual = openstack_upgrade.get_upgrade_groups() + pprint.pprint(expected) + pprint.pprint(actual) + self.assertEqual( + actual, + expected) + + def test_get_series_upgrade_groups(self): + expected = collections.OrderedDict([ + ('Core Identity', []), + ('Control Plane', ['cinder']), + ('Data Plane', ['nova-compute']), + ('sweep_up', ['mydb', 'ntp'])]) + actual = openstack_upgrade.get_series_upgrade_groups() + pprint.pprint(expected) + pprint.pprint(actual) + self.assertEqual( + actual, + expected) + + def test_extract_charm_name_from_url(self): + self.assertEqual( + openstack_upgrade.extract_charm_name_from_url( + 'local:bionic/heat-12'), + 'heat') + self.assertEqual( + openstack_upgrade.extract_charm_name_from_url( + 'cs:bionic/heat-12'), + 'heat') + self.assertEqual( + openstack_upgrade.extract_charm_name_from_url('cs:heat'), + 'heat') diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index eb7add6..239681f 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -16,6 +16,7 @@ """Define class for Series Upgrade.""" +import asyncio import logging import os import unittest @@ -23,13 +24,30 @@ import unittest from zaza import model from zaza.openstack.utilities import ( cli as cli_utils, - generic as generic_utils, + series_upgrade as series_upgrade_utils, + upgrade_utils as upgrade_utils, ) from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest +def _filter_easyrsa(app, app_config, model_name=None): + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "easyrsa" in charm_name: + logging.warn("Skipping series upgrade of easyrsa Bug #1850121") + return True + return False + + +def _filter_etcd(app, app_config, model_name=None): + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "etcd" in charm_name: + logging.warn("Skipping series upgrade of easyrsa Bug #1850124") + return True + return False + + class SeriesUpgradeTest(unittest.TestCase): - """Class to encapsulate Sereis Upgrade Tests.""" + """Class to encapsulate Series Upgrade Tests.""" @classmethod def setUpClass(cls): @@ -47,77 +65,35 @@ class SeriesUpgradeTest(unittest.TestCase): applications = model.get_status().applications completed_machines = [] - for application in applications: - # Defaults - origin = "openstack-origin" - pause_non_leader_subordinate = True - pause_non_leader_primary = True - post_upgrade_functions = [] + for application, app_details in applications: # Skip subordinates - if applications[application]["subordinate-to"]: + if app_details["subordinate-to"]: continue - if "easyrsa" in applications[application]["charm"]: - logging.warn("Skipping series upgrade of easyrsa Bug #1850121") + if "easyrsa" in app_details["charm"]: + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850121") continue - if "etcd" in applications[application]["charm"]: - logging.warn("Skipping series upgrade of easyrsa Bug #1850124") + if "etcd" in app_details["charm"]: + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850124") continue - if "percona-cluster" in applications[application]["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = True - if "rabbitmq-server" in applications[application]["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = False - if "nova-compute" in applications[application]["charm"]: - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "ceph" in applications[application]["charm"]: - origin = "source" - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "designate-bind" in applications[application]["charm"]: - origin = None - if "tempest" in applications[application]["charm"]: - origin = None - if "memcached" in applications[application]["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "vault" in applications[application]["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = True - post_upgrade_functions = [ - ('zaza.openstack.charm_tests.vault.setup.' - 'mojo_unseal_by_unit')] - if "mongodb" in applications[application]["charm"]: - # Mongodb needs to run series upgrade - # on its secondaries first. - generic_utils.series_upgrade_non_leaders_first( - application, - from_series=self.from_series, - to_series=self.to_series, - completed_machines=completed_machines, - post_upgrade_functions=post_upgrade_functions) - continue - - # The rest are likley APIs use defaults - - generic_utils.series_upgrade_application( + charm_name = upgrade_utils.extract_charm_name_from_url( + app_details['charm']) + upgrade_config = series_upgrade_utils.app_config( + charm_name, + is_async=False) + upgrade_function = upgrade_config.pop('upgrade_function') + logging.warn("About to upgrade {}".format(application)) + upgrade_function( application, - pause_non_leader_primary=pause_non_leader_primary, - pause_non_leader_subordinate=pause_non_leader_subordinate, + **upgrade_config, from_series=self.from_series, to_series=self.to_series, - origin=origin, completed_machines=completed_machines, workaround_script=self.workaround_script, files=self.files, - post_upgrade_functions=post_upgrade_functions) - - if "rabbitmq-server" in applications[application]["charm"]: + ) + if "rabbitmq-server" in app_details["charm"]: logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( @@ -126,7 +102,7 @@ class SeriesUpgradeTest(unittest.TestCase): action_params={}) model.block_until_all_units_idle() - if "percona-cluster" in applications[application]["charm"]: + if "percona-cluster" in app_details["charm"]: logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( @@ -215,5 +191,95 @@ class XenialBionicSeriesUpgrade(SeriesUpgradeTest): cls.to_series = "bionic" +class ParallelSeriesUpgradeTest(unittest.TestCase): + """Class to encapsulate Series Upgrade Tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + cli_utils.setup_logging() + cls.from_series = None + cls.to_series = None + cls.workaround_script = None + cls.files = [] + + def test_200_run_series_upgrade(self): + """Run series upgrade.""" + # Set Feature Flag + os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + upgrade_groups = upgrade_utils.get_series_upgrade_groups( + extra_filters=[_filter_etcd, _filter_easyrsa]) + applications = model.get_status().applications + completed_machines = [] + for group_name, group in upgrade_groups.items(): + logging.warn("About to upgrade {} ({})".format(group_name, group)) + upgrade_group = [] + for application, app_details in applications.items(): + if application not in group: + continue + charm_name = upgrade_utils.extract_charm_name_from_url( + app_details['charm']) + upgrade_config = series_upgrade_utils.app_config(charm_name) + upgrade_function = upgrade_config.pop('upgrade_function') + logging.warn("About to upgrade {}".format(application)) + upgrade_group.append( + upgrade_function( + application, + **upgrade_config, + from_series=self.from_series, + to_series=self.to_series, + completed_machines=completed_machines, + workaround_script=self.workaround_script, + files=self.files, + )) + asyncio.get_event_loop().run_until_complete( + asyncio.gather(*upgrade_group)) + if "rabbitmq-server" in group: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) + model.block_until_all_units_idle() + + if "percona-cluster" in group: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) + model.block_until_all_units_idle() + + +class ParallelTrustyXenialSeriesUpgrade(ParallelSeriesUpgradeTest): + """Trusty to Xenial Series Upgrade. + + Makes no assumptions about what is in the deployment. + """ + + @classmethod + def setUpClass(cls): + """Run setup for Trusty to Xenial Series Upgrades.""" + super(ParallelTrustyXenialSeriesUpgrade, cls).setUpClass() + cls.from_series = "trusty" + cls.to_series = "xenial" + + +class ParallelXenialBionicSeriesUpgrade(ParallelSeriesUpgradeTest): + """Xenial to Bionic Series Upgrade. + + Makes no assumptions about what is in the deployment. + """ + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(ParallelXenialBionicSeriesUpgrade, cls).setUpClass() + cls.from_series = "xenial" + cls.to_series = "bionic" + + if __name__ == "__main__": unittest.main() diff --git a/zaza/openstack/utilities/charm_upgrade.py b/zaza/openstack/utilities/charm_upgrade.py new file mode 100644 index 0000000..017b642 --- /dev/null +++ b/zaza/openstack/utilities/charm_upgrade.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of functions to support charm upgrade testing.""" diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index c0566b0..3a22404 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -14,6 +14,7 @@ """Collection of functions that did not fit anywhere else.""" +import asyncio import logging import os import socket @@ -25,11 +26,6 @@ from zaza import model from zaza.openstack.utilities import juju as juju_utils from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE -from zaza.charm_lifecycle import utils as cl_utils - -SUBORDINATE_PAUSE_RESUME_BLACKLIST = [ - "cinder-ceph", -] def dict_to_yaml(dict_data): @@ -192,255 +188,6 @@ def get_yaml_config(config_file): return yaml.safe_load(open(config_file, 'r').read()) -def run_post_upgrade_functions(post_upgrade_functions): - """Execute list supplied functions. - - :param post_upgrade_functions: List of functions - :type post_upgrade_functions: [function, function, ...] - """ - if post_upgrade_functions: - for func in post_upgrade_functions: - logging.info("Running {}".format(func)) - cl_utils.get_class(func)() - - -def series_upgrade_non_leaders_first(application, from_series="trusty", - to_series="xenial", - completed_machines=[], - post_upgrade_functions=None): - """Series upgrade non leaders first. - - Wrap all the functionality to handle series upgrade for charms - which must have non leaders upgraded first. - - :param application: Name of application to upgrade series - :type application: str - :param from_series: The series from which to upgrade - :type from_series: str - :param to_series: The series to which to upgrade - :type to_series: str - :param completed_machines: List of completed machines which do no longer - require series upgrade. - :type completed_machines: list - :returns: None - :rtype: None - """ - status = model.get_status().applications[application] - leader = None - non_leaders = [] - for unit in status["units"]: - if status["units"][unit].get("leader"): - leader = unit - else: - non_leaders.append(unit) - - # Series upgrade the non-leaders first - for unit in non_leaders: - machine = status["units"][unit]["machine"] - if machine not in completed_machines: - logging.info("Series upgrade non-leader unit: {}" - .format(unit)) - series_upgrade(unit, machine, - from_series=from_series, to_series=to_series, - origin=None, - post_upgrade_functions=post_upgrade_functions) - run_post_upgrade_functions(post_upgrade_functions) - completed_machines.append(machine) - else: - logging.info("Skipping unit: {}. Machine: {} already upgraded. " - .format(unit, machine, application)) - model.block_until_all_units_idle() - - # Series upgrade the leader - machine = status["units"][leader]["machine"] - logging.info("Series upgrade leader: {}".format(leader)) - if machine not in completed_machines: - series_upgrade(leader, machine, - from_series=from_series, to_series=to_series, - origin=None, - post_upgrade_functions=post_upgrade_functions) - completed_machines.append(machine) - else: - logging.info("Skipping unit: {}. Machine: {} already upgraded." - .format(unit, machine, application)) - model.block_until_all_units_idle() - - -def series_upgrade_application(application, pause_non_leader_primary=True, - pause_non_leader_subordinate=True, - from_series="trusty", to_series="xenial", - origin='openstack-origin', - completed_machines=[], - files=None, workaround_script=None, - post_upgrade_functions=None): - """Series upgrade application. - - Wrap all the functionality to handle series upgrade for a given - application. Including pausing non-leader units. - - :param application: Name of application to upgrade series - :type application: str - :param pause_non_leader_primary: Whether the non-leader applications should - be paused - :type pause_non_leader_primary: bool - :param pause_non_leader_subordinate: Whether the non-leader subordinate - hacluster applications should be - paused - :type pause_non_leader_subordinate: bool - :param from_series: The series from which to upgrade - :type from_series: str - :param to_series: The series to which to upgrade - :type to_series: str - :param origin: The configuration setting variable name for changing origin - source. (openstack-origin or source) - :type origin: str - :param completed_machines: List of completed machines which do no longer - require series upgrade. - :type completed_machines: list - :param files: Workaround files to scp to unit under upgrade - :type files: list - :param workaround_script: Workaround script to run during series upgrade - :type workaround_script: str - :returns: None - :rtype: None - """ - status = model.get_status().applications[application] - - # For some applications (percona-cluster) the leader unit must upgrade - # first. For API applications the non-leader haclusters must be paused - # before upgrade. Finally, for some applications this is arbitrary but - # generalized. - leader = None - non_leaders = [] - for unit in status["units"]: - if status["units"][unit].get("leader"): - leader = unit - else: - non_leaders.append(unit) - - # Pause the non-leaders - for unit in non_leaders: - if pause_non_leader_subordinate: - if status["units"][unit].get("subordinates"): - for subordinate in status["units"][unit]["subordinates"]: - _app = subordinate.split('/')[0] - if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: - logging.info("Skipping pausing {} - blacklisted" - .format(subordinate)) - else: - logging.info("Pausing {}".format(subordinate)) - model.run_action( - subordinate, "pause", action_params={}) - if pause_non_leader_primary: - logging.info("Pausing {}".format(unit)) - model.run_action(unit, "pause", action_params={}) - - machine = status["units"][leader]["machine"] - # Series upgrade the leader - logging.info("Series upgrade leader: {}".format(leader)) - if machine not in completed_machines: - series_upgrade(leader, machine, - from_series=from_series, to_series=to_series, - origin=origin, workaround_script=workaround_script, - files=files, - post_upgrade_functions=post_upgrade_functions) - completed_machines.append(machine) - else: - logging.info("Skipping unit: {}. Machine: {} already upgraded." - "But setting origin on the application {}" - .format(unit, machine, application)) - logging.info("Set origin on {}".format(application)) - set_origin(application, origin) - model.block_until_all_units_idle() - - # Series upgrade the non-leaders - for unit in non_leaders: - machine = status["units"][unit]["machine"] - if machine not in completed_machines: - logging.info("Series upgrade non-leader unit: {}" - .format(unit)) - series_upgrade(unit, machine, - from_series=from_series, to_series=to_series, - origin=origin, workaround_script=workaround_script, - files=files, - post_upgrade_functions=post_upgrade_functions) - completed_machines.append(machine) - else: - logging.info("Skipping unit: {}. Machine: {} already upgraded. " - "But setting origin on the application {}" - .format(unit, machine, application)) - logging.info("Set origin on {}".format(application)) - set_origin(application, origin) - model.block_until_all_units_idle() - - -def series_upgrade(unit_name, machine_num, - from_series="trusty", to_series="xenial", - origin='openstack-origin', - files=None, workaround_script=None, - post_upgrade_functions=None): - """Perform series upgrade on a unit. - - :param unit_name: Unit Name - :type unit_name: str - :param machine_num: Machine number - :type machine_num: str - :param from_series: The series from which to upgrade - :type from_series: str - :param to_series: The series to which to upgrade - :type to_series: str - :param origin: The configuration setting variable name for changing origin - source. (openstack-origin or source) - :type origin: str - :param files: Workaround files to scp to unit under upgrade - :type files: list - :param workaround_script: Workaround script to run during series upgrade - :type workaround_script: str - :returns: None - :rtype: None - """ - logging.info("Series upgrade {}".format(unit_name)) - application = unit_name.split('/')[0] - set_dpkg_non_interactive_on_unit(unit_name) - dist_upgrade(unit_name) - model.block_until_all_units_idle() - logging.info("Prepare series upgrade on {}".format(machine_num)) - model.prepare_series_upgrade(machine_num, to_series=to_series) - logging.info("Waiting for workload status 'blocked' on {}" - .format(unit_name)) - model.block_until_unit_wl_status(unit_name, "blocked") - logging.info("Waiting for model idleness") - model.block_until_all_units_idle() - wrap_do_release_upgrade(unit_name, from_series=from_series, - to_series=to_series, files=files, - workaround_script=workaround_script) - logging.info("Reboot {}".format(unit_name)) - reboot(unit_name) - logging.info("Waiting for workload status 'blocked' on {}" - .format(unit_name)) - model.block_until_unit_wl_status(unit_name, "blocked") - logging.info("Waiting for model idleness") - model.block_until_all_units_idle() - logging.info("Set origin on {}".format(application)) - # Allow for charms which have neither source nor openstack-origin - if origin: - set_origin(application, origin) - model.block_until_all_units_idle() - logging.info("Complete series upgrade on {}".format(machine_num)) - model.complete_series_upgrade(machine_num) - model.block_until_all_units_idle() - logging.info("Running run_post_upgrade_functions {}".format( - post_upgrade_functions)) - run_post_upgrade_functions(post_upgrade_functions) - logging.info("Waiting for workload status 'active' on {}" - .format(unit_name)) - model.block_until_unit_wl_status(unit_name, "active") - model.block_until_all_units_idle() - # This step may be performed by juju in the future - logging.info("Set series on {} to {}".format(application, to_series)) - model.set_series(application, to_series) - - def set_origin(application, origin='openstack-origin', pocket='distro'): """Set the configuration option for origin source. @@ -459,44 +206,23 @@ def set_origin(application, origin='openstack-origin', pocket='distro'): model.set_application_config(application, {origin: pocket}) -def wrap_do_release_upgrade(unit_name, from_series="trusty", - to_series="xenial", - files=None, workaround_script=None): - """Wrap do release upgrade. +async def async_set_origin(application, origin='openstack-origin', + pocket='distro'): + """Set the configuration option for origin source. - In a production environment this step would be run administratively. - For testing purposes we need this automated. - - :param unit_name: Unit Name - :type unit_name: str - :param from_series: The series from which to upgrade - :type from_series: str - :param to_series: The series to which to upgrade - :type to_series: str - :param files: Workaround files to scp to unit under upgrade - :type files: list - :param workaround_script: Workaround script to run during series upgrade - :type workaround_script: str + :param application: Name of application to upgrade series + :type application: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param pocket: Origin source cloud pocket. + i.e. 'distro' or 'cloud:xenial-newton' + :type pocket: str :returns: None :rtype: None """ - # Pre upgrade hacks - # There are a few necessary hacks to accomplish an automated upgrade - # to overcome some packaging bugs. - # Copy scripts - if files: - logging.info("SCP files") - for _file in files: - logging.info("SCP {}".format(_file)) - model.scp_to_unit(unit_name, _file, os.path.basename(_file)) - - # Run Script - if workaround_script: - logging.info("Running workaround script") - run_via_ssh(unit_name, workaround_script) - - # Actually do the do_release_upgrade - do_release_upgrade(unit_name) + logging.info("Set origin on {} to {}".format(application, origin)) + await model.async_set_application_config(application, {origin: pocket}) def run_via_ssh(unit_name, cmd): @@ -521,24 +247,28 @@ def run_via_ssh(unit_name, cmd): logging.warn(e) -def dist_upgrade(unit_name): - """Run dist-upgrade on unit after update package db. +async def async_run_via_ssh(unit_name, cmd, raise_exceptions=False): + """Run command on unit via ssh. + + For executing commands on units when the juju agent is down. :param unit_name: Unit Name - :type unit_name: str + :param cmd: Command to execute on remote unit + :type cmd: str :returns: None :rtype: None """ - logging.info('Updating package db ' + unit_name) - update_cmd = 'sudo apt update' - model.run_on_unit(unit_name, update_cmd) - - logging.info('Updating existing packages ' + unit_name) - dist_upgrade_cmd = ( - """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ - """-o "Dpkg::Options::=--force-confdef" """ - """-o "Dpkg::Options::=--force-confold" dist-upgrade""") - model.run_on_unit(unit_name, dist_upgrade_cmd) + if "sudo" not in cmd: + # cmd.insert(0, "sudo") + cmd = "sudo {}".format(cmd) + cmd = ['juju', 'ssh', unit_name, cmd] + try: + await check_call(cmd) + except subprocess.CalledProcessError as e: + logging.warn("Failed command {} on {}".format(cmd, unit_name)) + logging.warn(e) + if raise_exceptions: + raise e def check_commands_on_units(commands, units): @@ -566,26 +296,6 @@ def check_commands_on_units(commands, units): return None -def do_release_upgrade(unit_name): - """Run do-release-upgrade noninteractive. - - :param unit_name: Unit Name - :type unit_name: str - :returns: None - :rtype: None - """ - logging.info('Upgrading ' + unit_name) - # NOTE: It is necessary to run this via juju ssh rather than juju run due - # to timeout restrictions and error handling. - cmd = ['juju', 'ssh', unit_name, 'sudo', 'DEBIAN_FRONTEND=noninteractive', - 'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive'] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - logging.warn("Failed do-release-upgrade for {}".format(unit_name)) - logging.warn(e) - - def reboot(unit_name): """Reboot unit. @@ -604,6 +314,38 @@ def reboot(unit_name): pass +async def async_reboot(unit_name): + """Reboot unit. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + # NOTE: When used with series upgrade the agent will be down. + # Even juju run will not work + await async_run_via_ssh(unit_name, "sudo reboot && exit") + + +async def check_call(cmd): + """Asynchronous function to check a subprocess call. + + :param cmd: Command to execute + :type cmd: List[str] + :returns: None + :rtype: None + """ + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + logging.warn("STDOUT: {}".format(stdout)) + logging.warn("STDERR: {}".format(stderr)) + raise subprocess.CalledProcessError(proc.returncode, cmd) + + def set_dpkg_non_interactive_on_unit( unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"): """Set dpkg options on unit. @@ -620,6 +362,22 @@ def set_dpkg_non_interactive_on_unit( model.run_on_unit(unit_name, cmd) +async def async_set_dpkg_non_interactive_on_unit( + unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"): + """Set dpkg options on unit. + + :param unit_name: Unit Name + :type unit_name: str + :param apt_conf_d: Apt.conf file to update + :type apt_conf_d: str + """ + DPKG_NON_INTERACTIVE = 'DPkg::options { "--force-confdef"; };' + # Check if the option exists. If not, add it to the apt.conf.d file + cmd = ("grep '{option}' {file_name} || echo '{option}' >> {file_name}" + .format(option=DPKG_NON_INTERACTIVE, file_name=apt_conf_d)) + await model.async_run_on_unit(unit_name, cmd) + + def get_process_id_list(unit_name, process_name, expect_success=True): """Get a list of process ID(s). diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 3c4aa8f..9ad4d50 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -16,26 +16,14 @@ This module contains a number of functions for upgrading OpenStack. """ -import re import logging import zaza.openstack.utilities.juju as juju_utils import zaza.model from zaza import sync_wrapper - -SERVICE_GROUPS = { - 'Core Identity': ['keystone'], - 'Storage': [ - 'ceph-mon', 'ceph-osd', 'ceph-fs', 'ceph-radosgw', 'swift-proxy', - 'swift-storage'], - 'Control Plane': [ - 'aodh', 'barbican', 'ceilometer', 'cinder', 'designate', - 'designate-bind', 'glance', 'gnocchi', 'heat', 'manila', - 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', - 'nova-cloud-controller', 'openstack-dashboard'], - 'Compute': ['nova-compute']} - -UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] +from zaza.openstack.utilities.upgrade_utils import ( + get_upgrade_groups, +) async def async_pause_units(units, model_name=None): @@ -189,88 +177,6 @@ def set_upgrade_application_config(applications, new_source, model_name=model_name) -def _extract_charm_name_from_url(charm_url): - """Extract the charm name from the charm url. - - E.g. Extract 'heat' from local:bionic/heat-12 - - :param charm_url: Name of model to query. - :type charm_url: str - :returns: Charm name - :rtype: str - """ - charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) - return charm_name.split(':')[-1] - - -def get_upgrade_candidates(model_name=None): - """Extract list of apps from model that can be upgraded. - - :param model_name: Name of model to query. - :type model_name: str - :returns: List of application that can have their payload upgraded. - :rtype: [] - """ - status = zaza.model.get_status(model_name=model_name) - candidates = {} - for app, app_config in status.applications.items(): - # Filter out subordinates - if app_config.get("subordinate-to"): - logging.warning( - "Excluding {} from upgrade, it is a subordinate".format(app)) - continue - - # Filter out charms on the naughty list - charm_name = _extract_charm_name_from_url(app_config['charm']) - if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: - logging.warning( - "Excluding {} from upgrade, on the exclude list".format(app)) - continue - - # Filter out charms that have no source option - charm_options = zaza.model.get_application_config( - app, model_name=model_name).keys() - src_options = ['openstack-origin', 'source'] - if not [x for x in src_options if x in charm_options]: - logging.warning( - "Excluding {} from upgrade, no src option".format(app)) - continue - - candidates[app] = app_config - return candidates - - -def get_upgrade_groups(model_name=None): - """Place apps in the model into their upgrade groups. - - Place apps in the model into their upgrade groups. If an app is deployed - but is not in SERVICE_GROUPS then it is placed in a sweep_up group. - - :param model_name: Name of model to query. - :type model_name: str - :returns: Dict of group lists keyed on group name. - :rtype: {} - """ - apps_in_model = get_upgrade_candidates(model_name=model_name) - - groups = {} - for phase_name, charms in SERVICE_GROUPS.items(): - group = [] - for app, app_config in apps_in_model.items(): - charm_name = _extract_charm_name_from_url(app_config['charm']) - if charm_name in charms: - group.append(app) - groups[phase_name] = group - - sweep_up = [] - for app in apps_in_model: - if not (app in [a for group in groups.values() for a in group]): - sweep_up.append(app) - - groups['sweep_up'] = sweep_up - return groups - - def is_action_upgradable(app, model_name=None): """Can application be upgraded using action managed upgrade method. diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py new file mode 100644 index 0000000..866e019 --- /dev/null +++ b/zaza/openstack/utilities/series_upgrade.py @@ -0,0 +1,855 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of functions for testing series upgrade.""" + +import collections +import copy +import concurrent +import logging +import os + +from zaza import model +from zaza.charm_lifecycle import utils as cl_utils +import zaza.openstack.utilities.generic as os_utils + + +SUBORDINATE_PAUSE_RESUME_BLACKLIST = [ + "cinder-ceph", +] + + +def app_config(charm_name, is_async=True): + """Return a dict with the upgrade config for an application. + + :param charm_name: Name of the charm about to upgrade + :type charm_name: str + :param async: Whether the upgreade functions should be async + :type async: bool + :returns: A dicitonary of the upgrade config for the application + :rtype: Dict + """ + if is_async: + default_upgrade = async_series_upgrade_application + secondary_first_upgrade = async_series_upgrade_non_leaders_first + else: + default_upgrade = series_upgrade_application + secondary_first_upgrade = series_upgrade_non_leaders_first + default = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'upgrade_function': default_upgrade, + 'post_upgrade_functions': []} + _app_settings = collections.defaultdict(lambda: default) + ceph = { + 'origin': "source", + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + } + exceptions = { + 'rabbitmq-server': { + 'origin': 'source', + 'pause_non_leader_subordinate': False, }, + 'percona-cluster': {'origin': 'source', }, + 'nova-compute': { + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, }, + 'ceph': ceph, + 'ceph-mon': ceph, + 'ceph-osd': ceph, + 'designate-bind': {'origin': None, }, + 'tempest': {'origin': None, }, + 'memcached': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + }, + 'vault': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': True, + 'post_upgrade_functions': [ + ('zaza.openstack.charm_tests.vault.setup.' + 'mojo_unseal_by_unit')] + }, + 'mongodb': { + 'upgrade_function': secondary_first_upgrade, + } + + } + for key, value in exceptions.items(): + _app_settings[key] = copy.deepcopy(default) + _app_settings[key].update(value) + return _app_settings[charm_name] + + +def run_post_upgrade_functions(post_upgrade_functions): + """Execute list supplied functions. + + :param post_upgrade_functions: List of functions + :type post_upgrade_functions: [function, function, ...] + """ + if post_upgrade_functions: + for func in post_upgrade_functions: + logging.info("Running {}".format(func)) + cl_utils.get_class(func)() + + +def series_upgrade_non_leaders_first(application, from_series="trusty", + to_series="xenial", + completed_machines=[], + post_upgrade_functions=None): + """Series upgrade non leaders first. + + Wrap all the functionality to handle series upgrade for charms + which must have non leaders upgraded first. + + :param application: Name of application to upgrade series + :type application: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :returns: None + :rtype: None + """ + status = model.get_status().applications[application] + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Series upgrade the non-leaders first + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + series_upgrade(unit, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + run_post_upgrade_functions(post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + .format(unit, machine, application)) + model.block_until_all_units_idle() + + # Series upgrade the leader + machine = status["units"][leader]["machine"] + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + series_upgrade(leader, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + .format(unit, machine, application)) + model.block_until_all_units_idle() + + +async def async_series_upgrade_non_leaders_first(application, + from_series="trusty", + to_series="xenial", + completed_machines=[], + post_upgrade_functions=None): + """Series upgrade non leaders first. + + Wrap all the functionality to handle series upgrade for charms + which must have non leaders upgraded first. + + :param application: Name of application to upgrade series + :type application: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :returns: None + :rtype: None + """ + status = (await model.async_get_status()).applications[application] + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Series upgrade the non-leaders first + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + await async_series_upgrade( + unit, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + run_post_upgrade_functions(post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + .format(unit, machine, application)) + await model.async_block_until_all_units_idle() + + # Series upgrade the leader + machine = status["units"][leader]["machine"] + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + await async_series_upgrade( + leader, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + .format(unit, machine, application)) + await model.async_block_until_all_units_idle() + + +def series_upgrade_application(application, pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + from_series="trusty", to_series="xenial", + origin='openstack-origin', + completed_machines=[], + files=None, workaround_script=None, + post_upgrade_functions=None): + """Series upgrade application. + + Wrap all the functionality to handle series upgrade for a given + application. Including pausing non-leader units. + + :param application: Name of application to upgrade series + :type application: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + status = model.get_status().applications[application] + + # For some applications (percona-cluster) the leader unit must upgrade + # first. For API applications the non-leader haclusters must be paused + # before upgrade. Finally, for some applications this is arbitrary but + # generalized. + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Pause the non-leaders + for unit in non_leaders: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + _app = subordinate.split('/')[0] + if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: + logging.info("Skipping pausing {} - blacklisted" + .format(subordinate)) + else: + logging.info("Pausing {}".format(subordinate)) + model.run_action( + subordinate, "pause", action_params={}) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + model.run_action(unit, "pause", action_params={}) + + machine = status["units"][leader]["machine"] + # Series upgrade the leader + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + series_upgrade(leader, machine, + from_series=from_series, to_series=to_series, + origin=origin, workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + os_utils.set_origin(application, origin) + model.block_until_all_units_idle() + + # Series upgrade the non-leaders + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + series_upgrade(unit, machine, + from_series=from_series, to_series=to_series, + origin=origin, workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + os_utils.set_origin(application, origin) + model.block_until_all_units_idle() + + +async def async_series_upgrade_application( + application, + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + from_series="trusty", + to_series="xenial", + origin='openstack-origin', + completed_machines=None, + files=None, workaround_script=None, + post_upgrade_functions=None, + post_application_upgrade_functions=None): + """Series upgrade application. + + Wrap all the functionality to handle series upgrade for a given + application. Including pausing non-leader units. + + :param application: Name of application to upgrade series + :type application: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :param post_upgrade_functions: A list of functions to call after upgrading + each unit of an application + :type post_upgrade_functions: List[fn] + :param post_application_upgrade_functions: A list of functions to call + once after updating all units + of an application + :type post_application_upgrade_functions: List[fn] + :returns: None + :rtype: None + """ + if completed_machines is None: + completed_machines = [] + status = (await model.async_get_status()).applications[application] + + # For some applications (percona-cluster) the leader unit must upgrade + # first. For API applications the non-leader haclusters must be paused + # before upgrade. Finally, for some applications this is arbitrary but + # generalized. + leader = None + non_leaders = [] + logging.info("Configuring leader / non leaders for {}".format(application)) + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Pause the non-leaders + for unit in non_leaders: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + _app = subordinate.split('/')[0] + if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: + logging.info("Skipping pausing {} - blacklisted" + .format(subordinate)) + else: + logging.info("Pausing {}".format(subordinate)) + await model.async_run_action( + subordinate, "pause", action_params={}) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + await model.async_run_action(unit, "pause", action_params={}) + + machine = status["units"][leader]["machine"] + # Series upgrade the leader + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + await async_series_upgrade( + leader, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + await os_utils.async_set_origin(application, origin) + await wait_for_unit_idle(unit) + + # Series upgrade the non-leaders + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + await async_series_upgrade( + unit, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + await os_utils.async_set_origin(application, origin) + await wait_for_unit_idle(unit) + run_post_upgrade_functions(post_application_upgrade_functions) + + +# TODO: Move these functions into zaza.model +async def wait_for_unit_idle(unit_name, timeout=600): + """Wait until the unit's agent is idle. + + :param unit_name: The unit name of the application, ex: mysql/0 + :type unit_name: str + :param timeout: How long to wait before timing out + :type timeout: int + :returns: None + :rtype: None + """ + app = unit_name.split('/')[0] + try: + await model.async_block_until( + _unit_idle(app, unit_name), + timeout=timeout) + except concurrent.futures._base.TimeoutError: + raise model.ModelTimeout("Zaza has timed out waiting on {} to " + "reach idle state.".format(unit_name)) + + +def _unit_idle(app, unit_name): + async def f(): + x = await get_agent_status(app, unit_name) + return x == "idle" + return f + + +async def get_agent_status(app, unit_name): + """Get the current status of the specified unit. + + :param app: The name of the Juju application, ex: mysql + :type app: str + :param unit_name: The unit name of the application, ex: mysql/0 + :type unit_name: str + :returns: The agent status, either active / idle, returned by Juju + :rtype: str + """ + return (await model.async_get_status()). \ + applications[app]['units'][unit_name]['agent-status']['status'] + + +def series_upgrade(unit_name, machine_num, + from_series="trusty", to_series="xenial", + origin='openstack-origin', + files=None, workaround_script=None, + post_upgrade_functions=None): + """Perform series upgrade on a unit. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + logging.info("Series upgrade {}".format(unit_name)) + application = unit_name.split('/')[0] + os_utils.set_dpkg_non_interactive_on_unit(unit_name) + dist_upgrade(unit_name) + model.block_until_all_units_idle() + logging.info("Prepare series upgrade on {}".format(machine_num)) + model.prepare_series_upgrade(machine_num, to_series=to_series) + logging.info("Waiting for workload status 'blocked' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "blocked") + logging.info("Waiting for model idleness") + model.block_until_all_units_idle() + wrap_do_release_upgrade(unit_name, from_series=from_series, + to_series=to_series, files=files, + workaround_script=workaround_script) + logging.info("Reboot {}".format(unit_name)) + os_utils.reboot(unit_name) + logging.info("Waiting for workload status 'blocked' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "blocked") + logging.info("Waiting for model idleness") + model.block_until_all_units_idle() + logging.info("Set origin on {}".format(application)) + # Allow for charms which have neither source nor openstack-origin + if origin: + os_utils.set_origin(application, origin) + model.block_until_all_units_idle() + logging.info("Complete series upgrade on {}".format(machine_num)) + model.complete_series_upgrade(machine_num) + model.block_until_all_units_idle() + logging.info("Running run_post_upgrade_functions {}".format( + post_upgrade_functions)) + run_post_upgrade_functions(post_upgrade_functions) + logging.info("Waiting for workload status 'active' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "active") + model.block_until_all_units_idle() + # This step may be performed by juju in the future + logging.info("Set series on {} to {}".format(application, to_series)) + model.set_series(application, to_series) + + +async def async_series_upgrade(unit_name, machine_num, + from_series="trusty", to_series="xenial", + origin='openstack-origin', + files=None, workaround_script=None, + post_upgrade_functions=None): + """Perform series upgrade on a unit. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + logging.info("Series upgrade {}".format(unit_name)) + application = unit_name.split('/')[0] + await os_utils.async_set_dpkg_non_interactive_on_unit(unit_name) + await async_dist_upgrade(unit_name) + await wait_for_unit_idle(unit_name) + logging.info("Prepare series upgrade on {}".format(machine_num)) + await async_prepare_series_upgrade(machine_num, to_series=to_series) + logging.info("Waiting for workload status 'blocked' on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status(unit_name, "blocked") + logging.info("Waiting for unit {} idleness".format(unit_name)) + await wait_for_unit_idle(unit_name) + await async_wrap_do_release_upgrade(unit_name, from_series=from_series, + to_series=to_series, files=files, + workaround_script=workaround_script) + logging.info("Reboot {}".format(unit_name)) + await os_utils.async_reboot(unit_name) + logging.info("Waiting for workload status 'blocked' on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status(unit_name, "blocked") + # Allow for charms which have neither source nor openstack-origin + if origin: + logging.info("Set origin on {}".format(application)) + await os_utils.async_set_origin(application, origin) + await wait_for_unit_idle(unit_name) + logging.info("Complete series upgrade on {}".format(machine_num)) + await async_complete_series_upgrade(machine_num) + await wait_for_unit_idle(unit_name, timeout=1200) + logging.info("Running run_post_upgrade_functions {}".format( + post_upgrade_functions)) + run_post_upgrade_functions(post_upgrade_functions) + logging.info("Waiting for workload status 'active' on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status(unit_name, "active") + await wait_for_unit_idle(unit_name) + # This step may be performed by juju in the future + logging.info("Set series on {} to {}".format(application, to_series)) + await async_set_series(application, to_series) + + +async def async_prepare_series_upgrade(machine_num, to_series="xenial"): + """Execute juju series-upgrade prepare on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + :param machine_num: Machine number + :type machine_num: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = await model.async_get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + machine_num, "prepare", to_series, "--yes"] + logging.info("About to call '{}'".format(cmd)) + await os_utils.check_call(cmd) + + +async def async_complete_series_upgrade(machine_num): + """Execute juju series-upgrade complete on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + :param machine_num: Machine number + :type machine_num: str + :returns: None + :rtype: None + """ + juju_model = await model.async_get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + machine_num, "complete"] + await os_utils.check_call(cmd) + + +async def async_set_series(application, to_series): + """Execute juju set-series complete on application. + + NOTE: This is a new feature in juju and not yet in libjuju. + :param application: Name of application to upgrade series + :type application: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = await model.async_get_juju_model() + cmd = ["juju", "set-series", "-m", juju_model, + application, to_series] + await os_utils.check_call(cmd) + + +def wrap_do_release_upgrade(unit_name, from_series="trusty", + to_series="xenial", + files=None, workaround_script=None): + """Wrap do release upgrade. + + In a production environment this step would be run administratively. + For testing purposes we need this automated. + + :param unit_name: Unit Name + :type unit_name: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + # Pre upgrade hacks + # There are a few necessary hacks to accomplish an automated upgrade + # to overcome some packaging bugs. + # Copy scripts + if files: + logging.info("SCP files") + for _file in files: + logging.info("SCP {}".format(_file)) + model.scp_to_unit(unit_name, _file, os.path.basename(_file)) + + # Run Script + if workaround_script: + logging.info("Running workaround script") + os_utils.run_via_ssh(unit_name, workaround_script) + + # Actually do the do_release_upgrade + do_release_upgrade(unit_name) + + +async def async_wrap_do_release_upgrade(unit_name, from_series="trusty", + to_series="xenial", + files=None, workaround_script=None): + """Wrap do release upgrade. + + In a production environment this step would be run administratively. + For testing purposes we need this automated. + + :param unit_name: Unit Name + :type unit_name: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + # Pre upgrade hacks + # There are a few necessary hacks to accomplish an automated upgrade + # to overcome some packaging bugs. + # Copy scripts + if files: + logging.info("SCP files") + for _file in files: + logging.info("SCP {}".format(_file)) + await model.async_scp_to_unit( + unit_name, _file, os.path.basename(_file)) + + # Run Script + if workaround_script: + logging.info("Running workaround script") + await os_utils.async_run_via_ssh(unit_name, workaround_script) + + # Actually do the do_release_upgrade + await async_do_release_upgrade(unit_name) + + +def dist_upgrade(unit_name): + """Run dist-upgrade on unit after update package db. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Updating package db ' + unit_name) + update_cmd = 'sudo apt update' + model.run_on_unit(unit_name, update_cmd) + + logging.info('Updating existing packages ' + unit_name) + dist_upgrade_cmd = ( + """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + model.run_on_unit(unit_name, dist_upgrade_cmd) + + +async def async_dist_upgrade(unit_name): + """Run dist-upgrade on unit after update package db. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Updating package db ' + unit_name) + update_cmd = 'sudo apt update' + await model.async_run_on_unit(unit_name, update_cmd) + + logging.info('Updating existing packages ' + unit_name) + dist_upgrade_cmd = ( + """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + await model.async_run_on_unit(unit_name, dist_upgrade_cmd) + + +def do_release_upgrade(unit_name): + """Run do-release-upgrade noninteractive. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Upgrading ' + unit_name) + # NOTE: It is necessary to run this via juju ssh rather than juju run due + # to timeout restrictions and error handling. + os_utils.run_via_ssh( + unit_name, + 'DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive') + + +async def async_do_release_upgrade(unit_name): + """Run do-release-upgrade noninteractive. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Upgrading ' + unit_name) + # NOTE: It is necessary to run this via juju ssh rather than juju run due + # to timeout restrictions and error handling. + await os_utils.async_run_via_ssh( + unit_name, + 'DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive', + raise_exceptions=True) diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py new file mode 100644 index 0000000..062e9b1 --- /dev/null +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -0,0 +1,184 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of functions to support upgrade testing.""" +import re +import logging +import collections +import zaza.model + + +SERVICE_GROUPS = collections.OrderedDict([ + ('Core Identity', ['keystone']), + ('Control Plane', [ + 'aodh', 'barbican', 'ceilometer', 'ceph-mon', 'ceph-fs', + 'ceph-radosgw', 'cinder', 'designate', + 'designate-bind', 'glance', 'gnocchi', 'heat', 'manila', + 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', + 'nova-cloud-controller', 'openstack-dashboard']), + ('Data Plane', [ + 'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage']) +]) + +UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] + + +def get_upgrade_candidates(model_name=None, filters=None): + """Extract list of apps from model that can be upgraded. + + :param model_name: Name of model to query. + :type model_name: str + :param filters: List of filter functions to apply + :type filters: List[fn] + :returns: List of application that can have their payload upgraded. + :rtype: [] + """ + if filters is None: + filters = [] + status = zaza.model.get_status(model_name=model_name) + candidates = {} + for app, app_config in status.applications.items(): + if _include_app(app, app_config, filters, model_name=model_name): + candidates[app] = app_config + return candidates + + +def _include_app(app, app_config, filters, model_name=None): + for filt in filters: + if filt(app, app_config, model_name=model_name): + return False + return True + + +def _filter_subordinates(app, app_config, model_name=None): + if app_config.get("subordinate-to"): + logging.warning( + "Excluding {} from upgrade, it is a subordinate".format(app)) + return True + return False + + +def _filter_openstack_upgrade_list(app, app_config, model_name=None): + charm_name = extract_charm_name_from_url(app_config['charm']) + if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: + print("Excluding {} from upgrade, on the exclude list".format(app)) + logging.warning( + "Excluding {} from upgrade, on the exclude list".format(app)) + return True + return False + + +def _filter_non_openstack_services(app, app_config, model_name=None): + charm_options = zaza.model.get_application_config( + app, model_name=model_name).keys() + src_options = ['openstack-origin', 'source'] + if not [x for x in src_options if x in charm_options]: + logging.warning( + "Excluding {} from upgrade, no src option".format(app)) + return True + return False + + +def get_upgrade_groups(model_name=None, extra_filters=None): + """Place apps in the model into their upgrade groups. + + Place apps in the model into their upgrade groups. If an app is deployed + but is not in SERVICE_GROUPS then it is placed in a sweep_up group. + + :param model_name: Name of model to query. + :type model_name: str + :returns: Dict of group lists keyed on group name. + :rtype: collections.OrderedDict + """ + filters = [ + _filter_subordinates, + _filter_openstack_upgrade_list, + _filter_non_openstack_services, + ] + if extra_filters: + if isinstance(extra_filters, list): + filters.extend(extra_filters) + elif callable(extra_filters): + filters.append(extra_filters) + else: + raise RuntimeError( + "extra_filters should be a list of " + "callables") + apps_in_model = get_upgrade_candidates( + model_name=model_name, + filters=filters,) + + return _build_service_groups(apps_in_model) + + +def get_series_upgrade_groups(model_name=None, extra_filters=None): + """Place apps in the model into their upgrade groups. + + Place apps in the model into their upgrade groups. If an app is deployed + but is not in SERVICE_GROUPS then it is placed in a sweep_up group. + + :param model_name: Name of model to query. + :type model_name: str + :returns: Dict of group lists keyed on group name. + :rtype: collections.OrderedDict + """ + filters = [_filter_subordinates] + if extra_filters: + if isinstance(extra_filters, list): + filters.extend(extra_filters) + elif callable(extra_filters): + filters.append(extra_filters) + else: + raise RuntimeError( + "extra_filters should be a list of " + "callables") + apps_in_model = get_upgrade_candidates( + model_name=model_name, + filters=filters) + + return _build_service_groups(apps_in_model) + + +def _build_service_groups(applications): + groups = collections.OrderedDict() + for phase_name, charms in SERVICE_GROUPS.items(): + group = [] + for app, app_config in applications.items(): + charm_name = extract_charm_name_from_url(app_config['charm']) + if charm_name in charms: + group.append(app) + groups[phase_name] = group + + sweep_up = [] + for app in applications: + if not (app in [a for group in groups.values() for a in group]): + sweep_up.append(app) + groups['sweep_up'] = sweep_up + for name, group in groups.items(): + group.sort() + return groups + + +def extract_charm_name_from_url(charm_url): + """Extract the charm name from the charm url. + + E.g. Extract 'heat' from local:bionic/heat-12 + + :param charm_url: Name of model to query. + :type charm_url: str + :returns: Charm name + :rtype: str + """ + charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) + return charm_name.split(':')[-1]