diff --git a/unit_tests/test_zaza_charm_lifecycle_prepare.py b/unit_tests/test_zaza_charm_lifecycle_prepare.py index 52d89bc..ce7a21e 100644 --- a/unit_tests/test_zaza_charm_lifecycle_prepare.py +++ b/unit_tests/test_zaza_charm_lifecycle_prepare.py @@ -75,7 +75,6 @@ class TestCharmLifecyclePrepare(ut_utils.BaseTestCase): self.add_model.assert_called_once_with( 'newmodel', config={ - 'agent-stream': 'proposed', 'default-series': 'xenial', 'image-stream': 'daily', 'test-mode': 'true', diff --git a/unit_tests/test_zaza_model.py b/unit_tests/test_zaza_model.py index 9771a41..fe567b9 100644 --- a/unit_tests/test_zaza_model.py +++ b/unit_tests/test_zaza_model.py @@ -112,6 +112,25 @@ class TestModel(ut_utils.BaseTestCase): } self.Model_mock = mock.MagicMock() + # Juju Status Object and data + self.key = "instance-id" + self.key_data = "machine-uuid" + self.machine = "1" + self.machine_data = {self.key: self.key_data} + self.unit = "app/1" + self.unit_data = { + "workload-status": {"status": "active"}, + "machine": self.machine} + self.application = "app" + self.application_data = {"units": {self.unit: self.unit_data}} + self.subordinate_application = "subordinate_application" + self.subordinate_application_data = { + "subordinate-to": [self.application]} + self.juju_status = mock.MagicMock() + self.juju_status.applications = { + self.application: self.application_data} + self.juju_status.machines = self.machine_data + async def _connect_model(model_name): return model_name @@ -886,34 +905,49 @@ disk_formats = ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar def test_block_until_unit_wl_status(self): async def _block_until(f, timeout=None): - if not f(): + rc = await f() + if not rc: raise asyncio.futures.TimeoutError - self.patch_object(model, 'get_juju_model', return_value='mname') + + async def _get_status(): + return self.juju_status + self.patch_object(model, 'Model') self.Model.return_value = self.Model_mock - self.Model_mock.block_until.side_effect = _block_until + self.patch_object(model, 'get_juju_model', return_value='mname') self.patch_object(model, 'get_unit_from_name') - self.get_unit_from_name.return_value = mock.MagicMock( - workload_status='active') + self.patch_object(model, 'async_get_status') + self.async_get_status.side_effect = _get_status + self.patch_object(model, 'async_block_until') + self.async_block_until.side_effect = _block_until model.block_until_unit_wl_status( - 'app/2', + 'app/1', 'active', timeout=0.1) def test_block_until_unit_wl_status_fail(self): async def _block_until(f, timeout=None): - if not f(): + rc = await f() + if not rc: raise asyncio.futures.TimeoutError - self.patch_object(model, 'get_juju_model', return_value='mname') + + async def _get_status(): + return self.juju_status + + (self.juju_status.applications[self.application] + ["units"][self.unit]["workload-status"]["status"]) = "blocked" + self.patch_object(model, 'Model') self.Model.return_value = self.Model_mock - self.Model_mock.block_until.side_effect = _block_until + self.patch_object(model, 'get_juju_model', return_value='mname') self.patch_object(model, 'get_unit_from_name') - self.get_unit_from_name.return_value = mock.MagicMock( - workload_status='maintenance') + self.patch_object(model, 'async_get_status') + self.async_get_status.side_effect = _get_status + self.patch_object(model, 'async_block_until') + self.async_block_until.side_effect = _block_until with self.assertRaises(asyncio.futures.TimeoutError): model.block_until_unit_wl_status( - 'app/2', + 'app/1', 'active', timeout=0.1) @@ -940,6 +974,38 @@ disk_formats = ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar with self.assertRaises(asyncio.futures.TimeoutError): model.wait_for_agent_status(timeout=0.1) + def test_prepare_series_upgrade(self): + self.patch_object(model, 'subprocess') + self.patch_object(model, 'get_juju_model', + return_value=self.model_name) + _machine_num = "1" + _to_series = "bionic" + model.prepare_series_upgrade(_machine_num, to_series=_to_series) + self.subprocess.check_call.assert_called_once_with( + ["juju", "upgrade-series", "-m", self.model_name, + "prepare", _machine_num, _to_series, "--agree"]) + + def test_complete_series_upgrade(self): + self.patch_object(model, 'get_juju_model', + return_value=self.model_name) + self.patch_object(model, 'subprocess') + _machine_num = "1" + model.complete_series_upgrade(_machine_num) + self.subprocess.check_call.assert_called_once_with( + ["juju", "upgrade-series", "-m", self.model_name, + "complete", _machine_num]) + + def test_set_series(self): + self.patch_object(model, 'get_juju_model', + return_value=self.model_name) + self.patch_object(model, 'subprocess') + _application = "application" + _to_series = "bionic" + model.set_series(_application, _to_series) + self.subprocess.check_call.assert_called_once_with( + ["juju", "set-series", "-m", self.model_name, + _application, _to_series]) + class AsyncModelTests(aiounittest.AsyncTestCase): diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 855e2d5..f44bf74 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -16,11 +16,42 @@ import mock import unit_tests.utils as ut_utils from zaza.utilities import generic as generic_utils +FAKE_STATUS = { + 'can-upgrade-to': '', + 'charm': 'local:trusty/app-136', + 'subordinate-to': [], + 'units': {'app/0': {'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}, + 'app/1': {'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': {'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}} + class TestGenericUtils(ut_utils.BaseTestCase): def setUp(self): super(TestGenericUtils, self).setUp() + # Patch all subprocess calls + self.patch( + 'zaza.utilities.generic.subprocess', + new_callable=mock.MagicMock(), + name='subprocess' + ) + + # Juju Status Object and data + self.juju_status = mock.MagicMock() + self.juju_status.applications.__getitem__.return_value = FAKE_STATUS + self.patch_object(generic_utils, "model") + self.model.get_status.return_value = self.juju_status def test_dict_to_yaml(self): _dict_data = {"key": "value"} @@ -133,3 +164,190 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.assertEqual(generic_utils.get_yaml_config(_filename), _yaml_dict) self._open.assert_called_once_with(_filename, "r") + + def test_do_release_upgrade(self): + _unit = "app/2" + generic_utils.do_release_upgrade(_unit) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'ssh', _unit, 'sudo', 'do-release-upgrade', + '-f', 'DistUpgradeViewNonInteractive']) + + def test_wrap_do_release_upgrade(self): + self.patch_object(generic_utils, "do_release_upgrade") + self.patch_object(generic_utils, "run_via_ssh") + self.patch_object(generic_utils.model, "scp_to_unit") + _unit = "app/2" + _from_series = "xenial" + _to_series = "bionic" + _workaround_script = "scriptname" + _files = ["filename", _workaround_script] + _scp_calls = [] + _run_calls = [ + mock.call(_unit, _workaround_script)] + for filename in _files: + _scp_calls.append(mock.call(_unit, filename, filename)) + generic_utils.wrap_do_release_upgrade( + _unit, to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) + self.scp_to_unit.assert_has_calls(_scp_calls) + self.run_via_ssh.assert_has_calls(_run_calls) + self.do_release_upgrade.assert_called_once_with(_unit) + + def test_reboot(self): + _unit = "app/2" + generic_utils.reboot(_unit) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'ssh', _unit, + 'sudo', 'reboot', '&&', 'exit']) + + def test_run_via_ssh(self): + _unit = "app/2" + _cmd = "hostname" + generic_utils.run_via_ssh(_unit, _cmd) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'ssh', _unit, + 'sudo ' + _cmd]) + + def test_set_origin(self): + "application, origin='openstack-origin', pocket='distro'):" + self.patch_object(generic_utils.model, "set_application_config") + _application = "application" + _origin = "source" + _pocket = "cloud:fake-cloud" + generic_utils.set_origin(_application, origin=_origin, pocket=_pocket) + self.set_application_config.assert_called_once_with( + _application, {_origin: _pocket}) + + def test_series_upgrade(self): + self.patch_object(generic_utils.model, "block_until_all_units_idle") + self.patch_object(generic_utils.model, "block_until_unit_wl_status") + self.patch_object(generic_utils.model, "prepare_series_upgrade") + self.patch_object(generic_utils.model, "complete_series_upgrade") + self.patch_object(generic_utils.model, "set_series") + self.patch_object(generic_utils, "set_origin") + self.patch_object(generic_utils, "wrap_do_release_upgrade") + self.patch_object(generic_utils, "reboot") + _unit = "app/2" + _application = "app" + _machine_num = "4" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + generic_utils.series_upgrade( + _unit, _machine_num, origin=_origin, + to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) + self.block_until_all_units_idle.called_with() + self.prepare_series_upgrade.assert_called_once_with( + _machine_num, to_series=_to_series) + self.wrap_do_release_upgrade.assert_called_once_with( + _unit, to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) + self.complete_series_upgrade.assert_called_once_with(_machine_num) + self.set_series.assert_called_once_with(_application, _to_series) + self.set_origin.assert_called_once_with(_application, _origin) + self.reboot.assert_called_once_with(_unit) + + def test_series_upgrade_application_pause_peers_and_subordinates(self): + self.patch_object(generic_utils.model, "run_action") + self.patch_object(generic_utils, "series_upgrade") + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + # Peers and Subordinates + _run_action_calls = [ + mock.call("{}-hacluster/1".format(_application), + "pause", action_params={}), + mock.call("{}/1".format(_application), "pause", action_params={}), + mock.call("{}-hacluster/2".format(_application), + "pause", action_params={}), + mock.call("{}/2".format(_application), "pause", action_params={}), + ] + _series_upgrade_calls = [] + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files), + ) + + # Pause primary peers and subordinates + generic_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + workaround_script=_workaround_script, files=_files), + self.run_action.assert_has_calls(_run_action_calls) + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_series_upgrade_application_pause_subordinates(self): + self.patch_object(generic_utils.model, "run_action") + self.patch_object(generic_utils, "series_upgrade") + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + # Subordinates only + _run_action_calls = [ + mock.call("{}-hacluster/1".format(_application), + "pause", action_params={}), + mock.call("{}-hacluster/2".format(_application), + "pause", action_params={}), + ] + _series_upgrade_calls = [] + + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files), + ) + + # Pause subordinates + generic_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=False, + pause_non_leader_subordinate=True, + workaround_script=_workaround_script, files=_files), + self.run_action.assert_has_calls(_run_action_calls) + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_series_upgrade_application_no_pause(self): + self.patch_object(generic_utils.model, "run_action") + self.patch_object(generic_utils, "series_upgrade") + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _series_upgrade_calls = [] + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files), + ) + + # No Pausiing + generic_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=False, + pause_non_leader_subordinate=False, + workaround_script=_workaround_script, files=_files) + self.run_action.assert_not_called() + self.series_upgrade.assert_has_calls(_series_upgrade_calls) diff --git a/unit_tests/utilities/test_zaza_utilities_juju.py b/unit_tests/utilities/test_zaza_utilities_juju.py index 2b395c8..4754b6c 100644 --- a/unit_tests/utilities/test_zaza_utilities_juju.py +++ b/unit_tests/utilities/test_zaza_utilities_juju.py @@ -42,6 +42,7 @@ class TestJujuUtils(ut_utils.BaseTestCase): # Model self.patch_object(juju_utils, "model") self.model_name = "model-name" + self.model.get_juju_model.return_value = self.model_name self.model.get_status.return_value = self.juju_status self.run_output = {"Code": "0", "Stderr": "", "Stdout": "RESULT"} self.error_run_output = {"Code": "1", "Stderr": "ERROR", "Stdout": ""} diff --git a/zaza/charm_lifecycle/prepare.py b/zaza/charm_lifecycle/prepare.py index 993d85b..51f9a96 100644 --- a/zaza/charm_lifecycle/prepare.py +++ b/zaza/charm_lifecycle/prepare.py @@ -25,7 +25,6 @@ import zaza.model MODEL_DEFAULTS = { # Model defaults from charm-test-infra # https://jujucharms.com/docs/2.1/models-config - 'agent-stream': 'proposed', 'default-series': 'xenial', 'image-stream': 'daily', 'test-mode': 'true', diff --git a/zaza/charm_tests/nova/tests.py b/zaza/charm_tests/nova/tests.py index 2038a60..f7ef1ff 100644 --- a/zaza/charm_tests/nova/tests.py +++ b/zaza/charm_tests/nova/tests.py @@ -112,7 +112,7 @@ class BaseGuestCreateTest(unittest.TestCase): class CirrosGuestCreateTest(BaseGuestCreateTest): """Tests to launch a cirros image.""" - def test_launch_small_cirros_instance(self): + def test_launch_small_instance(self): """Launch a cirros instance and test connectivity.""" self.launch_instance(glance_setup.CIRROS_IMAGE_NAME) @@ -120,6 +120,6 @@ class CirrosGuestCreateTest(BaseGuestCreateTest): class LTSGuestCreateTest(BaseGuestCreateTest): """Tests to launch a LTS image.""" - def test_launch_small_cirros_instance(self): - """Launch a cirros instance and test connectivity.""" + def test_launch_small_instance(self): + """Launch a Bionic instance and test connectivity.""" self.launch_instance(glance_setup.LTS_IMAGE_NAME) diff --git a/zaza/charm_tests/series_upgrade/__init__.py b/zaza/charm_tests/series_upgrade/__init__.py new file mode 100644 index 0000000..890d264 --- /dev/null +++ b/zaza/charm_tests/series_upgrade/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test series upgrade.""" diff --git a/zaza/charm_tests/series_upgrade/tests.py b/zaza/charm_tests/series_upgrade/tests.py new file mode 100644 index 0000000..c61d48a --- /dev/null +++ b/zaza/charm_tests/series_upgrade/tests.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define class for Series Upgrade.""" + +import logging +import os +import unittest + +from zaza import model +from zaza.utilities import ( + cli as cli_utils, + generic as generic_utils, +) +from zaza.charm_tests.nova.tests import LTSGuestCreateTest + + +class SeriesUpgradeTest(unittest.TestCase): + """Class to encapsulate Sereis Upgrade Tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + cli_utils.setup_logging() + cls.lts = LTSGuestCreateTest() + + def validate_pre_series_upgrade_cloud(self): + """Validate pre series upgrade.""" + logging.info("Validate pre-series-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + def test_200_run_series_upgrade(self): + """Run series upgrade.""" + # Set Feature Flag + os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + + # While there are packaging upgrade bugs we need to be cheeky and + # workaround by using the new package's version of files + workaround_script = "/home/ubuntu/package-workarounds.sh" + src_workaround_script = os.path.basename(workaround_script) + + files = [src_workaround_script, 'corosync', 'corosync.conf'] + + applications = model.get_status().applications + from_series = "trusty" + to_series = "xenial" + for application in applications: + # Defaults + origin = "openstack-origin" + pause_non_leader_subordinate = True + pause_non_leader_primary = False + # Skip subordinates + if applications[application]["subordinate-to"]: + continue + if "percona-cluster" in applications[application]["charm"]: + origin = "source" + pause_non_leader_primary = True + pause_non_leader_subordinate = True + if "rabbitmq-server" in applications[application]["charm"]: + origin = "source" + pause_non_leader_primary = True + pause_non_leader_subordinate = False + if "nova-compute" in applications[application]["charm"]: + pause_non_leader_primary = False + pause_non_leader_subordinate = False + # Place holder for Ceph applications + # The rest are likley APIs and use defaults + + generic_utils.series_upgrade_application( + application, + pause_non_leader_primary=pause_non_leader_primary, + pause_non_leader_subordinate=pause_non_leader_subordinate, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files) + + def validate_series_upgraded_cloud(self): + """Validate post series upgrade.""" + logging.info("Validate post-series-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + +if __name__ == "__main__": + unittest.main() diff --git a/zaza/model.py b/zaza/model.py index 612b702..181b214 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -1090,25 +1090,31 @@ async def async_block_until_unit_wl_status(unit_name, status, model_name=None, blocks until the given unit has the desired workload status:: block_until_unit_wl_status( - 'modelname', aunit, - 'active') + 'active' + model_name='modelname') - :param model_name: Name of model to query. - :type model_name: str - :param unit_name: Name of unit to run action on + NOTE: unit.workload_status was actually reporting the application workload + status. Using the full status output from model.get_status() gives us + unit by unit workload status. + + :param unit_name: Name of unit :type unit_name: str :param status: Status to wait for (active, maintenance etc) :type status: str + :param model_name: Name of model to query. + :type model_name: str :param timeout: Time to wait for unit to achieved desired status :type timeout: float """ - async with run_in_model(model_name) as model: - unit = get_unit_from_name(unit_name, model) - await model.block_until( - lambda: unit.workload_status == status, - timeout=timeout - ) + async def _unit_status(): + app = unit_name.split("/")[0] + model_status = await async_get_status() + return (model_status.applications[app]['units'][unit_name] + ['workload-status']['status'] == status) + + async with run_in_model(model_name): + await async_block_until(_unit_status, timeout=timeout) block_until_unit_wl_status = sync_wrapper( async_block_until_unit_wl_status) @@ -1177,3 +1183,62 @@ class UnitNotFound(Exception): msg = ('Unit: {} was not found in current model'. format(unit_name)) super(UnitNotFound, self).__init__(msg) + + +# NOTE: The following are series upgrade related functions which are new +# features in juju. We can migrate to libjuju calls when the feature +# stabilizes. +def prepare_series_upgrade(machine_num, to_series="xenial"): + """Execute juju series-upgrade prepare on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + + :param machine_num: Machine number + :type machine_num: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + "prepare", machine_num, to_series, "--agree"] + subprocess.check_call(cmd) + + +def complete_series_upgrade(machine_num): + """Execute juju series-upgrade complete on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + + :param machine_num: Machine number + :type machine_num: str + :returns: None + :rtype: None + """ + juju_model = get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + "complete", machine_num] + subprocess.check_call(cmd) + + +def set_series(application, to_series): + """Execute juju set-series complete on application. + + NOTE: This is a new feature in juju and not yet in libjuju. + + :param application: Name of application to upgrade series + :type application: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = get_juju_model() + cmd = ["juju", "set-series", "-m", juju_model, + application, to_series] + subprocess.check_call(cmd) diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index 1a32bea..b5a549c 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -16,6 +16,7 @@ import logging import os +import subprocess import yaml from zaza import model @@ -165,3 +166,252 @@ def get_yaml_config(config_file): # the pwd. logging.info('Using config %s' % (config_file)) return yaml.load(open(config_file, 'r').read()) + + +def series_upgrade_application(application, pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + from_series="trusty", to_series="xenial", + origin='openstack-origin', + files=None, workaround_script=None): + """Series upgrade application. + + Wrap all the functionality to handle series upgrade for a given + application. Including pausing non-leader units. + + :param application: Name of application to upgrade series + :type application: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + status = model.get_status().applications[application] + + # For some applications (percona-cluster) the leader unit must upgrade + # first. For API applications the non-leader haclusters must be paused + # before upgrade. Finally, for some applications this is aribtrary but + # generalized. + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Pause the non-leaders + for unit in non_leaders: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + logging.info("Pausing {}".format(subordinate)) + model.run_action(subordinate, "pause", action_params={}) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + model.run_action(unit, "pause", action_params={}) + + # Series upgrade the leader + logging.info("Series upgrade leader: {}".format(leader)) + series_upgrade(leader, status["units"][leader]["machine"], + from_series=from_series, to_series=to_series, + origin=origin, workaround_script=workaround_script, + files=files) + + # Series upgrade the non-leaders + for unit in non_leaders: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + series_upgrade(unit, status["units"][unit]["machine"], + from_series=from_series, to_series=to_series, + origin=origin, workaround_script=workaround_script, + files=files) + + +def series_upgrade(unit_name, machine_num, + from_series="trusty", to_series="xenial", + origin='openstack-origin', + files=None, workaround_script=None): + """Perform series upgrade on a unit. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + logging.info("Series upgrade {}".format(unit_name)) + application = unit_name.split('/')[0] + logging.info("Prepare series upgrade on {}".format(machine_num)) + model.prepare_series_upgrade(machine_num, to_series=to_series) + logging.info("Watiing for workload status 'unknown' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "unknown") + wrap_do_release_upgrade(unit_name, from_series=from_series, + to_series=to_series, files=files, + workaround_script=workaround_script) + logging.info("Reboot {}".format(unit_name)) + reboot(unit_name) + logging.info("Watiing for workload status 'blocked' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "blocked") + logging.info("Watiing for model idleness") + model.block_until_all_units_idle() + logging.info("Complete series upgrade on {}".format(machine_num)) + model.complete_series_upgrade(machine_num) + model.block_until_all_units_idle() + logging.info("Watiing for workload status 'active' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "active") + model.block_until_all_units_idle() + logging.info("Set origin on {}".format(application)) + set_origin(application, origin) + model.block_until_all_units_idle() + # This step may be performed by juju in the future + logging.info("Set series on {} to {}".format(application, to_series)) + model.set_series(application, to_series) + + +def set_origin(application, origin='openstack-origin', pocket='distro'): + """Set the configuration option for origin source. + + :param application: Name of application to upgrade series + :type application: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param pocket: Origin source cloud pocket. + i.e. 'distro' or 'cloud:xenial-newton' + :type pocket: str + :returns: None + :rtype: None + """ + logging.info("Set origin on {} to {}".format(application, origin)) + model.set_application_config(application, {origin: pocket}) + + +def wrap_do_release_upgrade(unit_name, from_series="trusty", + to_series="xenial", + files=None, workaround_script=None): + """Wrap do release upgrade. + + In a production environment this step would be run administratively. + For testing purposes we need this automated. + + :param unit_name: Unit Name + :type unit_name: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + # Pre upgrade hacks + # There are a few necessary hacks to accomplish an automated upgrade + # to overcome some packaging bugs. + # Copy scripts + if files: + logging.info("SCP files") + for _file in files: + logging.info("SCP {}".format(_file)) + model.scp_to_unit(unit_name, _file, os.path.basename(_file)) + + # Run Script + if workaround_script: + logging.info("Running workaround script") + run_via_ssh(unit_name, workaround_script) + + # Actually do the do_release_upgrade + do_release_upgrade(unit_name) + + +def run_via_ssh(unit_name, cmd): + """Run command on unit via ssh. + + For executing commands on units when the juju agent is down. + + :param unit_name: Unit Name + :param cmd: Command to execute on remote unit + :type cmd: str + :returns: None + :rtype: None + """ + if "sudo" not in cmd: + cmd = "sudo {}".format(cmd) + cmd = ['juju', 'ssh', unit_name, cmd] + logging.info("Running {} on {}".format(cmd, unit_name)) + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + logging.warn("Failed command {} on {}".format(cmd, unit_name)) + logging.warn(e) + + +def do_release_upgrade(unit_name): + """Run do-release-upgrade noninteractive. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Upgrading ' + unit_name) + # NOTE: It is necessary to run this via juju ssh rather than juju run due + # to timeout restrictions and error handling. + cmd = ['juju', 'ssh', unit_name, 'sudo', + 'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive'] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + logging.warn("Failed do-release-upgrade for {}".format(unit_name)) + logging.warn(e) + + +def reboot(unit_name): + """Reboot unit. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + # NOTE: When used with series upgrade the agent will be down. + # Even juju run will not work + cmd = ['juju', 'ssh', unit_name, 'sudo', 'reboot', '&&', 'exit'] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + logging.info(e) + pass