From 056047d93ce636bd7322566bbd94853b0f2a7c40 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 17 Aug 2018 10:27:14 -0700 Subject: [PATCH 01/14] Allow non-charm standalone tests --- zaza/charm_lifecycle/deploy.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/zaza/charm_lifecycle/deploy.py b/zaza/charm_lifecycle/deploy.py index 872bdf8..1abec01 100755 --- a/zaza/charm_lifecycle/deploy.py +++ b/zaza/charm_lifecycle/deploy.py @@ -33,11 +33,7 @@ VALID_ENVIRONMENT_KEY_PREFIXES = [ 'OS_', 'VIP_RANGE', ] -LOCAL_OVERLAY_TEMPLATE = """ -applications: - {{ charm_name }}: - charm: {{ charm_location }} -""" +LOCAL_OVERLAY_TEMPLATE = "" LOCAL_OVERLAY_TEMPLATE_NAME = 'local-charm-overlay.yaml' From be4eee176e0afbad1b08fd84682d3202e689ed48 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 17 Aug 2018 10:28:40 -0700 Subject: [PATCH 02/14] Place holder for future standalone series upgrade Pure standalone tests are not quite ready yet. But this is a place holder for the series upgrade tests. --- zaza/charm_tests/series_upgrade/__init__.py | 15 ++++ zaza/charm_tests/series_upgrade/tests.py | 90 +++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 zaza/charm_tests/series_upgrade/__init__.py create mode 100644 zaza/charm_tests/series_upgrade/tests.py diff --git a/zaza/charm_tests/series_upgrade/__init__.py b/zaza/charm_tests/series_upgrade/__init__.py new file mode 100644 index 0000000..890d264 --- /dev/null +++ b/zaza/charm_tests/series_upgrade/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test series upgrade.""" diff --git a/zaza/charm_tests/series_upgrade/tests.py b/zaza/charm_tests/series_upgrade/tests.py new file mode 100644 index 0000000..a45abff --- /dev/null +++ b/zaza/charm_tests/series_upgrade/tests.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define class for Series Upgrade.""" + +import logging +import os +import unittest + +from zaza import model +from zaza.utilities import ( + cli as cli_utils, + generic as generic_utils, +) +from zaza.charm_tests.nova.tests import LTSGuestCreateTest + + +class SeriesUpgradeTest(unittest.TestCase): + """Class to encapsulate Sereis Upgrade Tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + cli_utils.setup_logging() + cls.lts = LTSGuestCreateTest() + + def test_100_validate_pre_series_upgrade_cloud(self): + """Validate pre series upgrade.""" + logging.info("Validate pre-series-upgrade: Spin up LTS instance") + self.lts.test_launch_small_cirros_instance() + + def test_200_run_series_upgrade(self): + """Run series upgrade.""" + # Set Feature Flag + os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + + applications = model.get_status().applications + from_series = "trusty" + to_series = "xenial" + for application in applications: + # Defaults + origin = "openstack-origin" + pause_non_leader_subordinate = True + pause_non_leader_primary = False + # Skip subordinates + if applications[application]["subordinate-to"]: + continue + if "percona-cluster" in applications[application]["charm"]: + origin = "source" + pause_non_leader_primary = True + pause_non_leader_subordinate = True + if "rabbitmq-server" in applications[application]["charm"]: + origin = "source" + pause_non_leader_primary = True + pause_non_leader_subordinate = False + if "nova-compute" in applications[application]["charm"]: + pause_non_leader_primary = False + pause_non_leader_subordinate = False + # Place holder for Ceph applications + # The rest are likley APIs and use defaults + + generic_utils.series_upgrade.series_upgrade_application( + application, + pause_non_leader_primary=pause_non_leader_primary, + pause_non_leader_subordinate=pause_non_leader_subordinate, + from_series=from_series, + to_series=to_series, + origin=origin) + + def test_300_validate_series_upgraded_cloud(self): + """Validate post series upgrade.""" + logging.info("Validate post-series-upgrade: Spin up LTS instance") + self.lts.test_launch_small_cirros_instance() + + +if __name__ == "__main__": + unittest.main() From 30082c2311992d18a224e9d9c16ab562c114a2d0 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 17 Aug 2018 14:43:23 -0700 Subject: [PATCH 03/14] Series Upgrade Utilities for performing series upgrades. Place holder for future stand alone series upgrade zaza test. Small change to overlay template to allow stand alone zaza tests. --- .../utilities/test_zaza_utilities_generic.py | 195 +++++++++++++++++ .../utilities/test_zaza_utilities_juju.py | 35 +++ zaza/charm_tests/series_upgrade/tests.py | 2 +- zaza/utilities/generic.py | 200 ++++++++++++++++++ zaza/utilities/juju.py | 68 ++++++ 5 files changed, 499 insertions(+), 1 deletion(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 855e2d5..13eb0b6 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -16,11 +16,36 @@ import mock import unit_tests.utils as ut_utils from zaza.utilities import generic as generic_utils +FAKE_STATUS = { + 'can-upgrade-to': '', + 'charm': 'local:trusty/app-136', + 'subordinate-to': [], + 'units': {'app/0': {'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}, + 'app/1': {'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': {'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}} + class TestGenericUtils(ut_utils.BaseTestCase): def setUp(self): super(TestGenericUtils, self).setUp() + # Patch all subprocess calls + self.patch( + 'zaza.utilities.generic.subprocess', + new_callable=mock.MagicMock(), + name='subprocess' + ) def test_dict_to_yaml(self): _dict_data = {"key": "value"} @@ -133,3 +158,173 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.assertEqual(generic_utils.get_yaml_config(_filename), _yaml_dict) self._open.assert_called_once_with(_filename, "r") + + def test_do_release_upgrade(self): + _unit = "app/2" + generic_utils.do_release_upgrade(_unit) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'ssh', _unit, 'sudo', 'do-release-upgrade', + '-f', 'DistUpgradeViewNonInteractive']) + + def test_wrap_do_release_upgrade(self): + self.patch_object(generic_utils, "do_release_upgrade") + self.patch_object(generic_utils.model, "scp_to_unit") + self.patch_object(generic_utils.model, "run_on_unit") + _unit = "app/2" + _from_series = "xenial" + _to_series = "bionic" + _scp_calls = [] + _run_calls = [ + mock.call(_unit, "/home/ubuntu/package-workarounds.sh"), + mock.call(_unit, "juju-updateseries " + "--from-series={} --to-series={}" + .format(_from_series, _to_series))] + for filename in ["package-workarounds.sh", + "corosync", "corosync.conf"]: + _scp_calls.append(mock.call(_unit, filename, filename)) + generic_utils.wrap_do_release_upgrade( + _unit, to_series=_to_series, from_series=_from_series) + self.scp_to_unit.assert_has_calls(_scp_calls) + self.run_on_unit.assert_has_calls(_run_calls) + self.do_release_upgrade.assert_called_once_with(_unit) + + def test_reboot(self): + _unit = "app/2" + generic_utils.reboot(_unit) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'run', "--unit", _unit, + 'sudo', 'reboot', '&&', 'exit']) + + def test_set_origin(self): + "application, origin='openstack-origin', pocket='distro'):" + self.patch_object(generic_utils.model, "set_application_config") + _application = "application" + _origin = "source" + _pocket = "cloud:fake-cloud" + generic_utils.set_origin(_application, origin=_origin, pocket=_pocket) + self.set_application_config.assert_called_once_with( + _application, {_origin: _pocket}) + + def test_series_upgrade(self): + self.patch_object(generic_utils.time, "sleep") + self.patch_object(generic_utils.model, "block_until_all_units_idle") + self.patch_object(generic_utils.juju_utils, "prepare_series_upgrade") + self.patch_object(generic_utils.juju_utils, "complete_series_upgrade") + self.patch_object(generic_utils.juju_utils, "set_series") + self.patch_object(generic_utils.juju_utils, "update_series") + self.patch_object(generic_utils, "set_origin") + self.patch_object(generic_utils, "wrap_do_release_upgrade") + self.patch_object(generic_utils, "reboot") + _unit = "app/2" + _application = "app" + _machine_num = "4" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + generic_utils.series_upgrade( + _unit, _machine_num, origin=_origin, + to_series=_to_series, from_series=_from_series) + self.block_until_all_units_idle.called_with() + self.prepare_series_upgrade.assert_called_once_with( + _machine_num, to_series=_to_series) + self.wrap_do_release_upgrade.assert_called_once_with( + _unit, to_series=_to_series, from_series=_from_series) + self.complete_series_upgrade.assert_called_once_with(_machine_num) + self.set_series.assert_called_once_with(_application, _to_series) + self.update_series.assert_called_once_with(_machine_num, _to_series) + self.set_origin.assert_called_once_with(_application, _origin) + self.reboot.assert_called_once_with(_unit) + + def test_series_upgrade_application_pause_peers_and_subordinates(self): + self.patch_object(generic_utils.juju_utils, "get_application_status") + self.patch_object(generic_utils.model, "run_action") + self.patch_object(generic_utils, "series_upgrade") + self.get_application_status.return_value = FAKE_STATUS + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + # Peers and Subordinates + _run_action_calls = [ + mock.call("{}-hacluster/1".format(_application), + "pause", action_params={}), + mock.call("{}/1".format(_application), "pause", action_params={}), + mock.call("{}-hacluster/2".format(_application), + "pause", action_params={}), + mock.call("{}/2".format(_application), "pause", action_params={}), + ] + _series_upgrade_calls = [] + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series), + ) + + # Pause primary peers and subordinates + generic_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=True, + pause_non_leader_subordinate=True) + self.run_action.assert_has_calls(_run_action_calls) + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_series_upgrade_application_pause_subordinates(self): + self.patch_object(generic_utils.juju_utils, "get_application_status") + self.patch_object(generic_utils.model, "run_action") + self.patch_object(generic_utils, "series_upgrade") + self.get_application_status.return_value = FAKE_STATUS + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + # Subordinates only + _run_action_calls = [ + mock.call("{}-hacluster/1".format(_application), + "pause", action_params={}), + mock.call("{}-hacluster/2".format(_application), + "pause", action_params={}), + ] + _series_upgrade_calls = [] + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series), + ) + + # Pause subordinates + generic_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=False, + pause_non_leader_subordinate=True) + self.run_action.assert_has_calls(_run_action_calls) + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_series_upgrade_application_no_pause(self): + self.patch_object(generic_utils.juju_utils, "get_application_status") + self.patch_object(generic_utils.model, "run_action") + self.patch_object(generic_utils, "series_upgrade") + self.get_application_status.return_value = FAKE_STATUS + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _series_upgrade_calls = [] + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series), + ) + + # No Pausiing + generic_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=False, + pause_non_leader_subordinate=False) + self.run_action.assert_not_called() + self.series_upgrade.assert_has_calls(_series_upgrade_calls) diff --git a/unit_tests/utilities/test_zaza_utilities_juju.py b/unit_tests/utilities/test_zaza_utilities_juju.py index 2b395c8..e0ffa41 100644 --- a/unit_tests/utilities/test_zaza_utilities_juju.py +++ b/unit_tests/utilities/test_zaza_utilities_juju.py @@ -22,6 +22,13 @@ class TestJujuUtils(ut_utils.BaseTestCase): def setUp(self): super(TestJujuUtils, self).setUp() + # Patch all subprocess calls + self.patch( + 'zaza.utilities.juju.subprocess', + new_callable=mock.MagicMock(), + name='subprocess' + ) + # Juju Status Object and data self.key = "instance-id" self.key_data = "machine-uuid" @@ -256,3 +263,31 @@ class TestJujuUtils(ut_utils.BaseTestCase): key='series' ) self.assertEqual(expected, actual) + + def test_prepare_series_upgrade(self): + _machine_num = "1" + _to_series = "bionic" + juju_utils.prepare_series_upgrade(_machine_num, to_series=_to_series) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'upgrade-series', 'prepare', + _machine_num, _to_series, '--agree']) + + def test_complete_series_upgrade(self): + _machine_num = "1" + juju_utils.complete_series_upgrade(_machine_num) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'upgrade-series', 'complete', _machine_num]) + + def test_set_series(self): + _application = "application" + _to_series = "bionic" + juju_utils.set_series(_application, _to_series) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'set-series', _application, _to_series]) + + def test_update_series(self): + _machine_num = "1" + _to_series = "bionic" + juju_utils.update_series(_machine_num, _to_series) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'update-series', _machine_num, _to_series]) diff --git a/zaza/charm_tests/series_upgrade/tests.py b/zaza/charm_tests/series_upgrade/tests.py index a45abff..82b4555 100644 --- a/zaza/charm_tests/series_upgrade/tests.py +++ b/zaza/charm_tests/series_upgrade/tests.py @@ -72,7 +72,7 @@ class SeriesUpgradeTest(unittest.TestCase): # Place holder for Ceph applications # The rest are likley APIs and use defaults - generic_utils.series_upgrade.series_upgrade_application( + generic_utils.series_upgrade_application( application, pause_non_leader_primary=pause_non_leader_primary, pause_non_leader_subordinate=pause_non_leader_subordinate, diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index 1a32bea..64ec7bf 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -16,6 +16,8 @@ import logging import os +import subprocess +import time import yaml from zaza import model @@ -165,3 +167,201 @@ def get_yaml_config(config_file): # the pwd. logging.info('Using config %s' % (config_file)) return yaml.load(open(config_file, 'r').read()) + + +def series_upgrade_application(application, pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + from_series="trusty", to_series="xenial", + origin='openstack-origin'): + """Series upgrade application. + + Wrap all the functionality to handle series upgrade for a given + application. Including pausing non-leader units. + + :param application: Name of application to upgrade series + :type application: str + :param pause_non_leader_primary: Should the non-leader applications be + paused? + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Should the non-leader subordinate + hacluster applications be paused? + :type pause_non_leader_subordinate: bool + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :returns: None + :rtype: None + """ + status = juju_utils.get_application_status(application=application) + + # For some applications (percona-cluster) the leader unit must upgrade + # first. For API applications the non-leader haclusters must be paused + # before upgrade. Finally, for some applications this is aribtrary but + # generalized. + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Pause the non-leaders + for unit in non_leaders: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + logging.info("Pausing {}".format(subordinate)) + model.run_action(subordinate, "pause", action_params={}) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + model.run_action(unit, "pause", action_params={}) + + # Series upgrade the leader + logging.info("Series upgrade leader: {}".format(leader)) + series_upgrade(leader, status["units"][leader]["machine"], + from_series=from_series, to_series=to_series, + origin=origin) + + # Series upgrade the non-leaders + for unit in non_leaders: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + series_upgrade(unit, status["units"][unit]["machine"], + from_series=from_series, to_series=to_series, + origin=origin) + + +def series_upgrade(unit_name, machine_num, + from_series="trusty", to_series="xenial", + origin='openstack-origin'): + """Perform series upgrade on a unit. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :returns: None + :rtype: None + """ + application = unit_name.split('/')[0] + juju_utils.prepare_series_upgrade(machine_num, to_series=to_series) + logging.info("Watiing for model idleness") + model.block_until_all_units_idle() + wrap_do_release_upgrade(unit_name, from_series=from_series, + to_series=to_series) + reboot(unit_name) + # Without the sleep model.block_on_all_units_idle returns to early + logging.info("Sleeping after reboot...") + time.sleep(30) + model.block_until_all_units_idle() + juju_utils.complete_series_upgrade(machine_num) + model.block_until_all_units_idle() + juju_utils.update_series(machine_num, to_series) + juju_utils.set_series(application, to_series) + set_origin(application, origin) + model.block_until_all_units_idle() + + +def set_origin(application, origin='openstack-origin', pocket='distro'): + """Set the configuration option for origin source. + + :param application: Name of application to upgrade series + :type application: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param pocket: Origin source cloud pocket. + i.e. 'distro' or 'cloud:xenial-newton' + :type pocket: str + :returns: None + :rtype: None + """ + model.set_application_config(application, {origin: pocket}) + + +def wrap_do_release_upgrade(unit_name, from_series="trusty", + to_series="xenial"): + """Wrap do release upgrade. + + In a production environment this step would be run administratively. + For testing purposes we need this automated. + + :param unit_name: Unit Name + :type unit_name: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + # Pre upgrade hacks + # There are a few necessary hacks to accomplish an automated upgrade + # to overcome some packaging bugs. + # Copy scripts + _files = ["package-workarounds.sh", "corosync", "corosync.conf"] + for _file in _files: + model.scp_to_unit(unit_name, _file, _file) + + # Run Scripts + model.run_on_unit( + unit_name, "/home/ubuntu/package-workarounds.sh") + + # Actually do the do_release_upgrade + do_release_upgrade(unit_name) + + # Post upgrade hacks + # Juju may at some point in the future auotmate this step + model.run_on_unit( + unit_name, + "juju-updateseries --from-series={} --to-series={}" + .format(from_series, to_series)) + + +def do_release_upgrade(unit_name): + """Run do-release-upgrade noninteractive. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Upgrading ' + unit_name) + # NOTE: It is necessary to run this via juju ssh rather than juju run do to + # timeout restrictions and error handling. + cmd = ['juju', 'ssh', unit_name, 'sudo', + 'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive'] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + logging.warn("Failed do-release-upgrade for {}".format(unit_name)) + logging.warn(e) + + +def reboot(unit_name): + """Reboot unit. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + # NOTE: Runnig this via model.run_on_unit fails to exit properly + cmd = ['juju', 'run', '--unit', unit_name, 'sudo', 'reboot', '&&', 'exit'] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + logging.info(e) + pass diff --git a/zaza/utilities/juju.py b/zaza/utilities/juju.py index 6f3c792..cc0e278 100644 --- a/zaza/utilities/juju.py +++ b/zaza/utilities/juju.py @@ -16,6 +16,7 @@ """Module for interacting with juju.""" import os from pathlib import Path +import subprocess import yaml from zaza import ( @@ -258,3 +259,70 @@ def leader_get(application, key=''): return yaml.load(result.get('Stdout')) else: raise model.CommandRunFailed(cmd, result) + + +def prepare_series_upgrade(machine_num, to_series="xenial"): + """Execute juju series-upgrade prepare on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + + :param machine_num: Machine number + :type machine_num: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + cmd = ["juju", "upgrade-series", "prepare", + machine_num, to_series, "--agree"] + subprocess.check_call(cmd) + + +def complete_series_upgrade(machine_num): + """Execute juju series-upgrade complete on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + + :param machine_num: Machine number + :type machine_num: str + :returns: None + :rtype: None + """ + cmd = ["juju", "upgrade-series", "complete", machine_num] + subprocess.check_call(cmd) + + +def set_series(application, to_series): + """Execute juju set-series complete on application. + + NOTE: This is a new feature in juju and not yet in libjuju. + + :param application: Name of application to upgrade series + :type application: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + cmd = ["juju", "set-series", application, to_series] + subprocess.check_call(cmd) + + +def update_series(machine_num, to_series): + """Execute juju update-series complete on machine. + + NOTE: This is a new feature in juju and not yet in libjuju. + + :param machine_num: Machine number + :type machine_num: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + cmd = ["juju", "update-series", machine_num, to_series] + subprocess.check_call(cmd) From 4eb3c2e7440fd845c3f91e054ad61b1f98bfedb9 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 21 Aug 2018 11:35:24 -0700 Subject: [PATCH 04/14] Set model on subprocess juju cmds --- .../utilities/test_zaza_utilities_juju.py | 14 +++++++++----- zaza/utilities/juju.py | 17 ++++++++++++----- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_juju.py b/unit_tests/utilities/test_zaza_utilities_juju.py index e0ffa41..e5cd43b 100644 --- a/unit_tests/utilities/test_zaza_utilities_juju.py +++ b/unit_tests/utilities/test_zaza_utilities_juju.py @@ -49,6 +49,7 @@ class TestJujuUtils(ut_utils.BaseTestCase): # Model self.patch_object(juju_utils, "model") self.model_name = "model-name" + self.model.get_juju_model.return_value = self.model_name self.model.get_status.return_value = self.juju_status self.run_output = {"Code": "0", "Stderr": "", "Stdout": "RESULT"} self.error_run_output = {"Code": "1", "Stderr": "ERROR", "Stdout": ""} @@ -269,25 +270,28 @@ class TestJujuUtils(ut_utils.BaseTestCase): _to_series = "bionic" juju_utils.prepare_series_upgrade(_machine_num, to_series=_to_series) self.subprocess.check_call.assert_called_once_with( - ['juju', 'upgrade-series', 'prepare', - _machine_num, _to_series, '--agree']) + ["juju", "upgrade-series", "-m", self.model_name, + "prepare", _machine_num, _to_series, "--agree"]) def test_complete_series_upgrade(self): _machine_num = "1" juju_utils.complete_series_upgrade(_machine_num) self.subprocess.check_call.assert_called_once_with( - ['juju', 'upgrade-series', 'complete', _machine_num]) + ["juju", "upgrade-series", "-m", self.model_name, + "complete", _machine_num]) def test_set_series(self): _application = "application" _to_series = "bionic" juju_utils.set_series(_application, _to_series) self.subprocess.check_call.assert_called_once_with( - ['juju', 'set-series', _application, _to_series]) + ["juju", "set-series", "-m", self.model_name, + _application, _to_series]) def test_update_series(self): _machine_num = "1" _to_series = "bionic" juju_utils.update_series(_machine_num, _to_series) self.subprocess.check_call.assert_called_once_with( - ['juju', 'update-series', _machine_num, _to_series]) + ["juju", "update-series", "-m", self.model_name, + _machine_num, _to_series]) diff --git a/zaza/utilities/juju.py b/zaza/utilities/juju.py index cc0e278..642a3d4 100644 --- a/zaza/utilities/juju.py +++ b/zaza/utilities/juju.py @@ -275,8 +275,9 @@ def prepare_series_upgrade(machine_num, to_series="xenial"): :returns: None :rtype: None """ - cmd = ["juju", "upgrade-series", "prepare", - machine_num, to_series, "--agree"] + juju_model = model.get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + "prepare", machine_num, to_series, "--agree"] subprocess.check_call(cmd) @@ -292,7 +293,9 @@ def complete_series_upgrade(machine_num): :returns: None :rtype: None """ - cmd = ["juju", "upgrade-series", "complete", machine_num] + juju_model = model.get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + "complete", machine_num] subprocess.check_call(cmd) @@ -308,7 +311,9 @@ def set_series(application, to_series): :returns: None :rtype: None """ - cmd = ["juju", "set-series", application, to_series] + juju_model = model.get_juju_model() + cmd = ["juju", "set-series", "-m", juju_model, + application, to_series] subprocess.check_call(cmd) @@ -324,5 +329,7 @@ def update_series(machine_num, to_series): :returns: None :rtype: None """ - cmd = ["juju", "update-series", machine_num, to_series] + juju_model = model.get_juju_model() + cmd = ["juju", "update-series", "-m", juju_model, + machine_num, to_series] subprocess.check_call(cmd) From c80f057eb6f0fa3dd4d40e42242778cb8d2376e4 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 21 Aug 2018 11:57:05 -0700 Subject: [PATCH 05/14] Parameterize workarounds --- .../utilities/test_zaza_utilities_generic.py | 42 +++++++++++++------ zaza/utilities/generic.py | 40 +++++++++++++----- 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 13eb0b6..dc61736 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -173,17 +173,19 @@ class TestGenericUtils(ut_utils.BaseTestCase): _unit = "app/2" _from_series = "xenial" _to_series = "bionic" + _workaround_script = "scriptname" + _files = ["filename", _workaround_script] _scp_calls = [] _run_calls = [ - mock.call(_unit, "/home/ubuntu/package-workarounds.sh"), + mock.call(_unit, _workaround_script), mock.call(_unit, "juju-updateseries " "--from-series={} --to-series={}" .format(_from_series, _to_series))] - for filename in ["package-workarounds.sh", - "corosync", "corosync.conf"]: + for filename in _files: _scp_calls.append(mock.call(_unit, filename, filename)) generic_utils.wrap_do_release_upgrade( - _unit, to_series=_to_series, from_series=_from_series) + _unit, to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) self.scp_to_unit.assert_has_calls(_scp_calls) self.run_on_unit.assert_has_calls(_run_calls) self.do_release_upgrade.assert_called_once_with(_unit) @@ -221,14 +223,18 @@ class TestGenericUtils(ut_utils.BaseTestCase): _from_series = "xenial" _to_series = "bionic" _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" generic_utils.series_upgrade( _unit, _machine_num, origin=_origin, - to_series=_to_series, from_series=_from_series) + to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) self.block_until_all_units_idle.called_with() self.prepare_series_upgrade.assert_called_once_with( _machine_num, to_series=_to_series) self.wrap_do_release_upgrade.assert_called_once_with( - _unit, to_series=_to_series, from_series=_from_series) + _unit, to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) self.complete_series_upgrade.assert_called_once_with(_machine_num) self.set_series.assert_called_once_with(_application, _to_series) self.update_series.assert_called_once_with(_machine_num, _to_series) @@ -244,6 +250,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): _from_series = "xenial" _to_series = "bionic" _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" # Peers and Subordinates _run_action_calls = [ mock.call("{}-hacluster/1".format(_application), @@ -258,7 +266,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): _series_upgrade_calls.append( mock.call("{}/{}".format(_application, machine_num), machine_num, origin=_origin, - from_series=_from_series, to_series=_to_series), + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files), ) # Pause primary peers and subordinates @@ -266,7 +275,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): _application, origin=_origin, to_series=_to_series, from_series=_from_series, pause_non_leader_primary=True, - pause_non_leader_subordinate=True) + pause_non_leader_subordinate=True, + workaround_script=_workaround_script, files=_files), self.run_action.assert_has_calls(_run_action_calls) self.series_upgrade.assert_has_calls(_series_upgrade_calls) @@ -279,6 +289,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): _from_series = "xenial" _to_series = "bionic" _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" # Subordinates only _run_action_calls = [ mock.call("{}-hacluster/1".format(_application), @@ -291,7 +303,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): _series_upgrade_calls.append( mock.call("{}/{}".format(_application, machine_num), machine_num, origin=_origin, - from_series=_from_series, to_series=_to_series), + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files), ) # Pause subordinates @@ -299,7 +312,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): _application, origin=_origin, to_series=_to_series, from_series=_from_series, pause_non_leader_primary=False, - pause_non_leader_subordinate=True) + pause_non_leader_subordinate=True, + workaround_script=_workaround_script, files=_files), self.run_action.assert_has_calls(_run_action_calls) self.series_upgrade.assert_has_calls(_series_upgrade_calls) @@ -313,11 +327,14 @@ class TestGenericUtils(ut_utils.BaseTestCase): _to_series = "bionic" _origin = "source" _series_upgrade_calls = [] + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" for machine_num in ("0", "1", "2"): _series_upgrade_calls.append( mock.call("{}/{}".format(_application, machine_num), machine_num, origin=_origin, - from_series=_from_series, to_series=_to_series), + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files), ) # No Pausiing @@ -325,6 +342,7 @@ class TestGenericUtils(ut_utils.BaseTestCase): _application, origin=_origin, to_series=_to_series, from_series=_from_series, pause_non_leader_primary=False, - pause_non_leader_subordinate=False) + pause_non_leader_subordinate=False, + workaround_script=_workaround_script, files=_files) self.run_action.assert_not_called() self.series_upgrade.assert_has_calls(_series_upgrade_calls) diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index 64ec7bf..01502ef 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -172,7 +172,8 @@ def get_yaml_config(config_file): def series_upgrade_application(application, pause_non_leader_primary=True, pause_non_leader_subordinate=True, from_series="trusty", to_series="xenial", - origin='openstack-origin'): + origin='openstack-origin', + files=None, workaround_script=None): """Series upgrade application. Wrap all the functionality to handle series upgrade for a given @@ -193,6 +194,10 @@ def series_upgrade_application(application, pause_non_leader_primary=True, :param origin: The configuration setting variable name for changing origin source. (openstack-origin or source) :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str :returns: None :rtype: None """ @@ -225,7 +230,8 @@ def series_upgrade_application(application, pause_non_leader_primary=True, logging.info("Series upgrade leader: {}".format(leader)) series_upgrade(leader, status["units"][leader]["machine"], from_series=from_series, to_series=to_series, - origin=origin) + origin=origin, workaround_script=workaround_script, + files=files) # Series upgrade the non-leaders for unit in non_leaders: @@ -233,12 +239,14 @@ def series_upgrade_application(application, pause_non_leader_primary=True, .format(unit)) series_upgrade(unit, status["units"][unit]["machine"], from_series=from_series, to_series=to_series, - origin=origin) + origin=origin, workaround_script=workaround_script, + files=files) def series_upgrade(unit_name, machine_num, from_series="trusty", to_series="xenial", - origin='openstack-origin'): + origin='openstack-origin', + files=None, workaround_script=None): """Perform series upgrade on a unit. :param unit_name: Unit Name @@ -252,6 +260,10 @@ def series_upgrade(unit_name, machine_num, :param origin: The configuration setting variable name for changing origin source. (openstack-origin or source) :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str :returns: None :rtype: None """ @@ -260,7 +272,8 @@ def series_upgrade(unit_name, machine_num, logging.info("Watiing for model idleness") model.block_until_all_units_idle() wrap_do_release_upgrade(unit_name, from_series=from_series, - to_series=to_series) + to_series=to_series, files=files, + workaround_script=workaround_script) reboot(unit_name) # Without the sleep model.block_on_all_units_idle returns to early logging.info("Sleeping after reboot...") @@ -292,7 +305,8 @@ def set_origin(application, origin='openstack-origin', pocket='distro'): def wrap_do_release_upgrade(unit_name, from_series="trusty", - to_series="xenial"): + to_series="xenial", + files=None, workaround_script=None): """Wrap do release upgrade. In a production environment this step would be run administratively. @@ -304,6 +318,10 @@ def wrap_do_release_upgrade(unit_name, from_series="trusty", :type from_series: str :param to_series: The series to which to upgrade :type to_series: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str :returns: None :rtype: None """ @@ -311,13 +329,13 @@ def wrap_do_release_upgrade(unit_name, from_series="trusty", # There are a few necessary hacks to accomplish an automated upgrade # to overcome some packaging bugs. # Copy scripts - _files = ["package-workarounds.sh", "corosync", "corosync.conf"] - for _file in _files: - model.scp_to_unit(unit_name, _file, _file) + if files: + for _file in files: + model.scp_to_unit(unit_name, _file, os.path.basename(_file)) # Run Scripts - model.run_on_unit( - unit_name, "/home/ubuntu/package-workarounds.sh") + if workaround_script: + model.run_on_unit(unit_name, workaround_script) # Actually do the do_release_upgrade do_release_upgrade(unit_name) From 35f3f547e084ad2e8fa285777c22a5056bc29772 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 7 Sep 2018 15:40:51 -0700 Subject: [PATCH 06/14] Revert LOCAL_OVERLAY_TEMPLATE removal --- zaza/charm_lifecycle/deploy.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zaza/charm_lifecycle/deploy.py b/zaza/charm_lifecycle/deploy.py index 1abec01..872bdf8 100755 --- a/zaza/charm_lifecycle/deploy.py +++ b/zaza/charm_lifecycle/deploy.py @@ -33,7 +33,11 @@ VALID_ENVIRONMENT_KEY_PREFIXES = [ 'OS_', 'VIP_RANGE', ] -LOCAL_OVERLAY_TEMPLATE = "" +LOCAL_OVERLAY_TEMPLATE = """ +applications: + {{ charm_name }}: + charm: {{ charm_location }} +""" LOCAL_OVERLAY_TEMPLATE_NAME = 'local-charm-overlay.yaml' From 77c57df9c6186a115b9782d8e54375fac3c35dd2 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 7 Sep 2018 15:42:17 -0700 Subject: [PATCH 07/14] Must use the snap version of agents when using edge version of juju --- unit_tests/test_zaza_charm_lifecycle_prepare.py | 1 - zaza/charm_lifecycle/prepare.py | 1 - 2 files changed, 2 deletions(-) diff --git a/unit_tests/test_zaza_charm_lifecycle_prepare.py b/unit_tests/test_zaza_charm_lifecycle_prepare.py index 52d89bc..ce7a21e 100644 --- a/unit_tests/test_zaza_charm_lifecycle_prepare.py +++ b/unit_tests/test_zaza_charm_lifecycle_prepare.py @@ -75,7 +75,6 @@ class TestCharmLifecyclePrepare(ut_utils.BaseTestCase): self.add_model.assert_called_once_with( 'newmodel', config={ - 'agent-stream': 'proposed', 'default-series': 'xenial', 'image-stream': 'daily', 'test-mode': 'true', diff --git a/zaza/charm_lifecycle/prepare.py b/zaza/charm_lifecycle/prepare.py index 993d85b..51f9a96 100644 --- a/zaza/charm_lifecycle/prepare.py +++ b/zaza/charm_lifecycle/prepare.py @@ -25,7 +25,6 @@ import zaza.model MODEL_DEFAULTS = { # Model defaults from charm-test-infra # https://jujucharms.com/docs/2.1/models-config - 'agent-stream': 'proposed', 'default-series': 'xenial', 'image-stream': 'daily', 'test-mode': 'true', From fb67806fc6c14d04fc73dd395b34b71f0c3df946 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 7 Sep 2018 15:42:46 -0700 Subject: [PATCH 08/14] Use status output for block_until_unit_wl_status unit.workload_status was actually reporting the application workload status. Using the full status output from model.get_status() gives us unit by unit workload status. Updated block_until_unit_wl_status to use the full status output to correctly block per unit. --- unit_tests/test_zaza_model.py | 58 +++++++++++++++++++++++++++-------- zaza/model.py | 24 ++++++++------- 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/unit_tests/test_zaza_model.py b/unit_tests/test_zaza_model.py index 9771a41..b911888 100644 --- a/unit_tests/test_zaza_model.py +++ b/unit_tests/test_zaza_model.py @@ -112,6 +112,25 @@ class TestModel(ut_utils.BaseTestCase): } self.Model_mock = mock.MagicMock() + # Juju Status Object and data + self.key = "instance-id" + self.key_data = "machine-uuid" + self.machine = "1" + self.machine_data = {self.key: self.key_data} + self.unit = "app/1" + self.unit_data = { + "workload-status": {"status": "active"}, + "machine": self.machine} + self.application = "app" + self.application_data = {"units": {self.unit: self.unit_data}} + self.subordinate_application = "subordinate_application" + self.subordinate_application_data = { + "subordinate-to": [self.application]} + self.juju_status = mock.MagicMock() + self.juju_status.applications = { + self.application: self.application_data} + self.juju_status.machines = self.machine_data + async def _connect_model(model_name): return model_name @@ -886,34 +905,49 @@ disk_formats = ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar def test_block_until_unit_wl_status(self): async def _block_until(f, timeout=None): - if not f(): + rc = await f() + if not rc: raise asyncio.futures.TimeoutError - self.patch_object(model, 'get_juju_model', return_value='mname') + + async def _get_status(): + return self.juju_status + self.patch_object(model, 'Model') self.Model.return_value = self.Model_mock - self.Model_mock.block_until.side_effect = _block_until + self.patch_object(model, 'get_juju_model', return_value='mname') self.patch_object(model, 'get_unit_from_name') - self.get_unit_from_name.return_value = mock.MagicMock( - workload_status='active') + self.patch_object(model, 'async_get_status') + self.async_get_status.side_effect = _get_status + self.patch_object(model, 'async_block_until') + self.async_block_until.side_effect = _block_until model.block_until_unit_wl_status( - 'app/2', + 'app/1', 'active', timeout=0.1) def test_block_until_unit_wl_status_fail(self): async def _block_until(f, timeout=None): - if not f(): + rc = await f() + if not rc: raise asyncio.futures.TimeoutError - self.patch_object(model, 'get_juju_model', return_value='mname') + + async def _get_status(): + return self.juju_status + + (self.juju_status.applications[self.application] + ["units"][self.unit]["workload-status"]["status"]) = "blocked" + self.patch_object(model, 'Model') self.Model.return_value = self.Model_mock - self.Model_mock.block_until.side_effect = _block_until + self.patch_object(model, 'get_juju_model', return_value='mname') self.patch_object(model, 'get_unit_from_name') - self.get_unit_from_name.return_value = mock.MagicMock( - workload_status='maintenance') + self.patch_object(model, 'async_get_status') + self.async_get_status.side_effect = _get_status + self.patch_object(model, 'async_block_until') + self.async_block_until.side_effect = _block_until with self.assertRaises(asyncio.futures.TimeoutError): model.block_until_unit_wl_status( - 'app/2', + 'app/1', 'active', timeout=0.1) diff --git a/zaza/model.py b/zaza/model.py index 612b702..86e773a 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -1090,25 +1090,27 @@ async def async_block_until_unit_wl_status(unit_name, status, model_name=None, blocks until the given unit has the desired workload status:: block_until_unit_wl_status( - 'modelname', aunit, - 'active') + 'active' + model_name='modelname') - :param model_name: Name of model to query. - :type model_name: str - :param unit_name: Name of unit to run action on + :param unit_name: Name of unit :type unit_name: str :param status: Status to wait for (active, maintenance etc) :type status: str + :param model_name: Name of model to query. + :type model_name: str :param timeout: Time to wait for unit to achieved desired status :type timeout: float """ - async with run_in_model(model_name) as model: - unit = get_unit_from_name(unit_name, model) - await model.block_until( - lambda: unit.workload_status == status, - timeout=timeout - ) + async def _unit_status(): + app = unit_name.split("/")[0] + model_status = await async_get_status() + return (model_status.applications[app]['units'][unit_name] + ['workload-status']['status'] == status) + + async with run_in_model(model_name): + await async_block_until(_unit_status, timeout=timeout) block_until_unit_wl_status = sync_wrapper( async_block_until_unit_wl_status) From a83c3c4d2a084493dbc782fc745f69a5e5333168 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 7 Sep 2018 15:46:47 -0700 Subject: [PATCH 09/14] Pass upgrade workaround files for standalone test --- zaza/charm_tests/series_upgrade/tests.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/zaza/charm_tests/series_upgrade/tests.py b/zaza/charm_tests/series_upgrade/tests.py index 82b4555..79d652e 100644 --- a/zaza/charm_tests/series_upgrade/tests.py +++ b/zaza/charm_tests/series_upgrade/tests.py @@ -37,7 +37,7 @@ class SeriesUpgradeTest(unittest.TestCase): cli_utils.setup_logging() cls.lts = LTSGuestCreateTest() - def test_100_validate_pre_series_upgrade_cloud(self): + def validate_pre_series_upgrade_cloud(self): """Validate pre series upgrade.""" logging.info("Validate pre-series-upgrade: Spin up LTS instance") self.lts.test_launch_small_cirros_instance() @@ -47,6 +47,13 @@ class SeriesUpgradeTest(unittest.TestCase): # Set Feature Flag os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + # While there are packaging upgrade bugs we need to be cheeky and + # workaround by using the new package's version of files + workaround_script = "/home/ubuntu/package-workarounds.sh" + src_workaround_script = os.path.basename(workaround_script) + + files = [src_workaround_script, 'corosync', 'corosync.conf'] + applications = model.get_status().applications from_series = "trusty" to_series = "xenial" @@ -78,9 +85,11 @@ class SeriesUpgradeTest(unittest.TestCase): pause_non_leader_subordinate=pause_non_leader_subordinate, from_series=from_series, to_series=to_series, - origin=origin) + origin=origin, + workaround_script=workaround_script, + files=files) - def test_300_validate_series_upgraded_cloud(self): + def validate_series_upgraded_cloud(self): """Validate post series upgrade.""" logging.info("Validate post-series-upgrade: Spin up LTS instance") self.lts.test_launch_small_cirros_instance() From c4182197d4b2c11205c1e65a162779df3c634a6a Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 7 Sep 2018 15:47:08 -0700 Subject: [PATCH 10/14] Use block_until_unit_wl_status for upgrade timing Remove all calls to time.sleep and use block_until_unit_wl_status to manage the timing of the upgrade steps. Also use run_via_ssh to execute commands while the juju agents are down during a series upgrade. --- .../utilities/test_zaza_utilities_generic.py | 23 ++++--- zaza/utilities/generic.py | 66 ++++++++++++++----- 2 files changed, 63 insertions(+), 26 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index dc61736..76481fc 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -168,8 +168,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): def test_wrap_do_release_upgrade(self): self.patch_object(generic_utils, "do_release_upgrade") + self.patch_object(generic_utils, "run_via_ssh") self.patch_object(generic_utils.model, "scp_to_unit") - self.patch_object(generic_utils.model, "run_on_unit") _unit = "app/2" _from_series = "xenial" _to_series = "bionic" @@ -177,26 +177,31 @@ class TestGenericUtils(ut_utils.BaseTestCase): _files = ["filename", _workaround_script] _scp_calls = [] _run_calls = [ - mock.call(_unit, _workaround_script), - mock.call(_unit, "juju-updateseries " - "--from-series={} --to-series={}" - .format(_from_series, _to_series))] + mock.call(_unit, _workaround_script)] for filename in _files: _scp_calls.append(mock.call(_unit, filename, filename)) generic_utils.wrap_do_release_upgrade( _unit, to_series=_to_series, from_series=_from_series, workaround_script=_workaround_script, files=_files) self.scp_to_unit.assert_has_calls(_scp_calls) - self.run_on_unit.assert_has_calls(_run_calls) + self.run_via_ssh.assert_has_calls(_run_calls) self.do_release_upgrade.assert_called_once_with(_unit) def test_reboot(self): _unit = "app/2" generic_utils.reboot(_unit) self.subprocess.check_call.assert_called_once_with( - ['juju', 'run', "--unit", _unit, + ['juju', 'ssh', _unit, 'sudo', 'reboot', '&&', 'exit']) + def test_run_via_ssh(self): + _unit = "app/2" + _cmd = "hostname" + generic_utils.run_via_ssh(_unit, _cmd) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'ssh', _unit, + 'sudo ' + _cmd]) + def test_set_origin(self): "application, origin='openstack-origin', pocket='distro'):" self.patch_object(generic_utils.model, "set_application_config") @@ -208,12 +213,11 @@ class TestGenericUtils(ut_utils.BaseTestCase): _application, {_origin: _pocket}) def test_series_upgrade(self): - self.patch_object(generic_utils.time, "sleep") self.patch_object(generic_utils.model, "block_until_all_units_idle") + self.patch_object(generic_utils.model, "block_until_unit_wl_status") self.patch_object(generic_utils.juju_utils, "prepare_series_upgrade") self.patch_object(generic_utils.juju_utils, "complete_series_upgrade") self.patch_object(generic_utils.juju_utils, "set_series") - self.patch_object(generic_utils.juju_utils, "update_series") self.patch_object(generic_utils, "set_origin") self.patch_object(generic_utils, "wrap_do_release_upgrade") self.patch_object(generic_utils, "reboot") @@ -237,7 +241,6 @@ class TestGenericUtils(ut_utils.BaseTestCase): workaround_script=_workaround_script, files=_files) self.complete_series_upgrade.assert_called_once_with(_machine_num) self.set_series.assert_called_once_with(_application, _to_series) - self.update_series.assert_called_once_with(_machine_num, _to_series) self.set_origin.assert_called_once_with(_application, _origin) self.reboot.assert_called_once_with(_unit) diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index 01502ef..b001db5 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -17,7 +17,6 @@ import logging import os import subprocess -import time import yaml from zaza import model @@ -267,24 +266,39 @@ def series_upgrade(unit_name, machine_num, :returns: None :rtype: None """ + logging.info("Series upgrade {}".format(unit_name)) application = unit_name.split('/')[0] + logging.info("Prepare series upgrade on {}".format(machine_num)) juju_utils.prepare_series_upgrade(machine_num, to_series=to_series) logging.info("Watiing for model idleness") model.block_until_all_units_idle() + logging.info("Watiing for workload status 'unknown' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "unknown") wrap_do_release_upgrade(unit_name, from_series=from_series, to_series=to_series, files=files, workaround_script=workaround_script) - reboot(unit_name) - # Without the sleep model.block_on_all_units_idle returns to early - logging.info("Sleeping after reboot...") - time.sleep(30) model.block_until_all_units_idle() + logging.info("Reboot {}".format(unit_name)) + reboot(unit_name) + logging.info("Watiing for workload status 'blocked' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "blocked") + logging.info("Watiing for model idleness") + model.block_until_all_units_idle() + logging.info("Complete series upgrade on {}".format(machine_num)) juju_utils.complete_series_upgrade(machine_num) model.block_until_all_units_idle() - juju_utils.update_series(machine_num, to_series) - juju_utils.set_series(application, to_series) + logging.info("Watiing for workload status 'active' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "active") + model.block_until_all_units_idle() + logging.info("Set origin on {}".format(application)) set_origin(application, origin) model.block_until_all_units_idle() + # This step may be performed by juju in the future + logging.info("Set series on {} to {}".format(application, to_series)) + juju_utils.set_series(application, to_series) def set_origin(application, origin='openstack-origin', pocket='distro'): @@ -301,6 +315,7 @@ def set_origin(application, origin='openstack-origin', pocket='distro'): :returns: None :rtype: None """ + logging.info("Set origin on {} to {}".format(application, origin)) model.set_application_config(application, {origin: pocket}) @@ -330,22 +345,40 @@ def wrap_do_release_upgrade(unit_name, from_series="trusty", # to overcome some packaging bugs. # Copy scripts if files: + logging.info("SCP files") for _file in files: + logging.info("SCP {}".format(_file)) model.scp_to_unit(unit_name, _file, os.path.basename(_file)) # Run Scripts if workaround_script: - model.run_on_unit(unit_name, workaround_script) + logging.info("Running workaround script") + run_via_ssh(unit_name, workaround_script) # Actually do the do_release_upgrade do_release_upgrade(unit_name) - # Post upgrade hacks - # Juju may at some point in the future auotmate this step - model.run_on_unit( - unit_name, - "juju-updateseries --from-series={} --to-series={}" - .format(from_series, to_series)) + +def run_via_ssh(unit_name, cmd): + """Run command on unit via ssh. + + For executing commands on units when the juju agent is down. + + :param unit_name: Unit Name + :param cmd: Command to execute on remote unit + :type cmd: str + :returns: None + :rtype: None + """ + if "sudo" not in cmd: + cmd = "sudo {}".format(cmd) + cmd = ['juju', 'ssh', unit_name, cmd] + logging.info("Running {} on {}".format(cmd, unit_name)) + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + logging.warn("Failed command {} on {}".format(cmd, unit_name)) + logging.warn(e) def do_release_upgrade(unit_name): @@ -376,8 +409,9 @@ def reboot(unit_name): :returns: None :rtype: None """ - # NOTE: Runnig this via model.run_on_unit fails to exit properly - cmd = ['juju', 'run', '--unit', unit_name, 'sudo', 'reboot', '&&', 'exit'] + # NOTE: When used with series upgrade the agent will be down. + # Even juju run will not work + cmd = ['juju', 'ssh', unit_name, 'sudo', 'reboot', '&&', 'exit'] try: subprocess.check_call(cmd) except subprocess.CalledProcessError as e: From e893cc4b403883eb498f94776f33d721fb42d578 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 10 Sep 2018 15:20:04 -0700 Subject: [PATCH 11/14] Do not wait for idleness after pre-series-upgrade The pre-series-upgrade hook stops the juju agent. This can cause the juju wait for idleness to wait forever. Wait for the expected workload status instead. --- zaza/utilities/generic.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index b001db5..d7743b2 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -270,15 +270,12 @@ def series_upgrade(unit_name, machine_num, application = unit_name.split('/')[0] logging.info("Prepare series upgrade on {}".format(machine_num)) juju_utils.prepare_series_upgrade(machine_num, to_series=to_series) - logging.info("Watiing for model idleness") - model.block_until_all_units_idle() logging.info("Watiing for workload status 'unknown' on {}" .format(unit_name)) model.block_until_unit_wl_status(unit_name, "unknown") wrap_do_release_upgrade(unit_name, from_series=from_series, to_series=to_series, files=files, workaround_script=workaround_script) - model.block_until_all_units_idle() logging.info("Reboot {}".format(unit_name)) reboot(unit_name) logging.info("Watiing for workload status 'blocked' on {}" From 9e24be435e567f92bb0266c9734b5fd5fdb6ee55 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 11 Sep 2018 07:51:24 -0700 Subject: [PATCH 12/14] Fix launch_small_instance in charm_tests --- zaza/charm_tests/nova/tests.py | 6 +++--- zaza/charm_tests/series_upgrade/tests.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/charm_tests/nova/tests.py b/zaza/charm_tests/nova/tests.py index 2038a60..f7ef1ff 100644 --- a/zaza/charm_tests/nova/tests.py +++ b/zaza/charm_tests/nova/tests.py @@ -112,7 +112,7 @@ class BaseGuestCreateTest(unittest.TestCase): class CirrosGuestCreateTest(BaseGuestCreateTest): """Tests to launch a cirros image.""" - def test_launch_small_cirros_instance(self): + def test_launch_small_instance(self): """Launch a cirros instance and test connectivity.""" self.launch_instance(glance_setup.CIRROS_IMAGE_NAME) @@ -120,6 +120,6 @@ class CirrosGuestCreateTest(BaseGuestCreateTest): class LTSGuestCreateTest(BaseGuestCreateTest): """Tests to launch a LTS image.""" - def test_launch_small_cirros_instance(self): - """Launch a cirros instance and test connectivity.""" + def test_launch_small_instance(self): + """Launch a Bionic instance and test connectivity.""" self.launch_instance(glance_setup.LTS_IMAGE_NAME) diff --git a/zaza/charm_tests/series_upgrade/tests.py b/zaza/charm_tests/series_upgrade/tests.py index 79d652e..c61d48a 100644 --- a/zaza/charm_tests/series_upgrade/tests.py +++ b/zaza/charm_tests/series_upgrade/tests.py @@ -40,7 +40,7 @@ class SeriesUpgradeTest(unittest.TestCase): def validate_pre_series_upgrade_cloud(self): """Validate pre series upgrade.""" logging.info("Validate pre-series-upgrade: Spin up LTS instance") - self.lts.test_launch_small_cirros_instance() + self.lts.test_launch_small_instance() def test_200_run_series_upgrade(self): """Run series upgrade.""" @@ -92,7 +92,7 @@ class SeriesUpgradeTest(unittest.TestCase): def validate_series_upgraded_cloud(self): """Validate post series upgrade.""" logging.info("Validate post-series-upgrade: Spin up LTS instance") - self.lts.test_launch_small_cirros_instance() + self.lts.test_launch_small_instance() if __name__ == "__main__": From 8874fc7961280146486ea7077ea2c1e934789796 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 11 Sep 2018 07:52:43 -0700 Subject: [PATCH 13/14] Fix docstrings --- zaza/model.py | 4 ++++ zaza/utilities/generic.py | 15 ++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/zaza/model.py b/zaza/model.py index 86e773a..9670eb3 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -1094,6 +1094,10 @@ async def async_block_until_unit_wl_status(unit_name, status, model_name=None, 'active' model_name='modelname') + NOTE: unit.workload_status was actually reporting the application workload + status. Using the full status output from model.get_status() gives us + unit by unit workload status. + :param unit_name: Name of unit :type unit_name: str :param status: Status to wait for (active, maintenance etc) diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index d7743b2..74ddfe5 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -180,11 +180,12 @@ def series_upgrade_application(application, pause_non_leader_primary=True, :param application: Name of application to upgrade series :type application: str - :param pause_non_leader_primary: Should the non-leader applications be - paused? + :param pause_non_leader_primary: Whether the non-leader applications should + be paused :type pause_non_leader_primary: bool - :param pause_non_leader_subordinate: Should the non-leader subordinate - hacluster applications be paused? + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused :type pause_non_leader_subordinate: bool :param from_series: The series from which to upgrade :type from_series: str @@ -347,7 +348,7 @@ def wrap_do_release_upgrade(unit_name, from_series="trusty", logging.info("SCP {}".format(_file)) model.scp_to_unit(unit_name, _file, os.path.basename(_file)) - # Run Scripts + # Run Script if workaround_script: logging.info("Running workaround script") run_via_ssh(unit_name, workaround_script) @@ -387,8 +388,8 @@ def do_release_upgrade(unit_name): :rtype: None """ logging.info('Upgrading ' + unit_name) - # NOTE: It is necessary to run this via juju ssh rather than juju run do to - # timeout restrictions and error handling. + # NOTE: It is necessary to run this via juju ssh rather than juju run due + # to timeout restrictions and error handling. cmd = ['juju', 'ssh', unit_name, 'sudo', 'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive'] try: From c5fe6ce7232abdaea9fb5b7231cf33e3b61a6a83 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 11 Sep 2018 09:33:49 -0700 Subject: [PATCH 14/14] Move upgrade series functions to zaza.model Moving the upgrade series functions to zaza.model. Once the feature stabilizes we can add functionality to libjuju and remove the subprocess calls. --- unit_tests/test_zaza_model.py | 32 ++++++++ .../utilities/test_zaza_utilities_generic.py | 20 ++--- .../utilities/test_zaza_utilities_juju.py | 38 ---------- zaza/model.py | 59 +++++++++++++++ zaza/utilities/generic.py | 8 +- zaza/utilities/juju.py | 75 ------------------- 6 files changed, 106 insertions(+), 126 deletions(-) diff --git a/unit_tests/test_zaza_model.py b/unit_tests/test_zaza_model.py index b911888..fe567b9 100644 --- a/unit_tests/test_zaza_model.py +++ b/unit_tests/test_zaza_model.py @@ -974,6 +974,38 @@ disk_formats = ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar with self.assertRaises(asyncio.futures.TimeoutError): model.wait_for_agent_status(timeout=0.1) + def test_prepare_series_upgrade(self): + self.patch_object(model, 'subprocess') + self.patch_object(model, 'get_juju_model', + return_value=self.model_name) + _machine_num = "1" + _to_series = "bionic" + model.prepare_series_upgrade(_machine_num, to_series=_to_series) + self.subprocess.check_call.assert_called_once_with( + ["juju", "upgrade-series", "-m", self.model_name, + "prepare", _machine_num, _to_series, "--agree"]) + + def test_complete_series_upgrade(self): + self.patch_object(model, 'get_juju_model', + return_value=self.model_name) + self.patch_object(model, 'subprocess') + _machine_num = "1" + model.complete_series_upgrade(_machine_num) + self.subprocess.check_call.assert_called_once_with( + ["juju", "upgrade-series", "-m", self.model_name, + "complete", _machine_num]) + + def test_set_series(self): + self.patch_object(model, 'get_juju_model', + return_value=self.model_name) + self.patch_object(model, 'subprocess') + _application = "application" + _to_series = "bionic" + model.set_series(_application, _to_series) + self.subprocess.check_call.assert_called_once_with( + ["juju", "set-series", "-m", self.model_name, + _application, _to_series]) + class AsyncModelTests(aiounittest.AsyncTestCase): diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 76481fc..f44bf74 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -47,6 +47,12 @@ class TestGenericUtils(ut_utils.BaseTestCase): name='subprocess' ) + # Juju Status Object and data + self.juju_status = mock.MagicMock() + self.juju_status.applications.__getitem__.return_value = FAKE_STATUS + self.patch_object(generic_utils, "model") + self.model.get_status.return_value = self.juju_status + def test_dict_to_yaml(self): _dict_data = {"key": "value"} _str_data = "key: value\n" @@ -215,9 +221,9 @@ class TestGenericUtils(ut_utils.BaseTestCase): def test_series_upgrade(self): self.patch_object(generic_utils.model, "block_until_all_units_idle") self.patch_object(generic_utils.model, "block_until_unit_wl_status") - self.patch_object(generic_utils.juju_utils, "prepare_series_upgrade") - self.patch_object(generic_utils.juju_utils, "complete_series_upgrade") - self.patch_object(generic_utils.juju_utils, "set_series") + self.patch_object(generic_utils.model, "prepare_series_upgrade") + self.patch_object(generic_utils.model, "complete_series_upgrade") + self.patch_object(generic_utils.model, "set_series") self.patch_object(generic_utils, "set_origin") self.patch_object(generic_utils, "wrap_do_release_upgrade") self.patch_object(generic_utils, "reboot") @@ -245,10 +251,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.reboot.assert_called_once_with(_unit) def test_series_upgrade_application_pause_peers_and_subordinates(self): - self.patch_object(generic_utils.juju_utils, "get_application_status") self.patch_object(generic_utils.model, "run_action") self.patch_object(generic_utils, "series_upgrade") - self.get_application_status.return_value = FAKE_STATUS _application = "app" _from_series = "xenial" _to_series = "bionic" @@ -284,10 +288,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.series_upgrade.assert_has_calls(_series_upgrade_calls) def test_series_upgrade_application_pause_subordinates(self): - self.patch_object(generic_utils.juju_utils, "get_application_status") self.patch_object(generic_utils.model, "run_action") self.patch_object(generic_utils, "series_upgrade") - self.get_application_status.return_value = FAKE_STATUS _application = "app" _from_series = "xenial" _to_series = "bionic" @@ -302,6 +304,7 @@ class TestGenericUtils(ut_utils.BaseTestCase): "pause", action_params={}), ] _series_upgrade_calls = [] + for machine_num in ("0", "1", "2"): _series_upgrade_calls.append( mock.call("{}/{}".format(_application, machine_num), @@ -321,10 +324,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.series_upgrade.assert_has_calls(_series_upgrade_calls) def test_series_upgrade_application_no_pause(self): - self.patch_object(generic_utils.juju_utils, "get_application_status") self.patch_object(generic_utils.model, "run_action") self.patch_object(generic_utils, "series_upgrade") - self.get_application_status.return_value = FAKE_STATUS _application = "app" _from_series = "xenial" _to_series = "bionic" @@ -332,6 +333,7 @@ class TestGenericUtils(ut_utils.BaseTestCase): _series_upgrade_calls = [] _files = ["filename", "scriptname"] _workaround_script = "scriptname" + for machine_num in ("0", "1", "2"): _series_upgrade_calls.append( mock.call("{}/{}".format(_application, machine_num), diff --git a/unit_tests/utilities/test_zaza_utilities_juju.py b/unit_tests/utilities/test_zaza_utilities_juju.py index e5cd43b..4754b6c 100644 --- a/unit_tests/utilities/test_zaza_utilities_juju.py +++ b/unit_tests/utilities/test_zaza_utilities_juju.py @@ -22,13 +22,6 @@ class TestJujuUtils(ut_utils.BaseTestCase): def setUp(self): super(TestJujuUtils, self).setUp() - # Patch all subprocess calls - self.patch( - 'zaza.utilities.juju.subprocess', - new_callable=mock.MagicMock(), - name='subprocess' - ) - # Juju Status Object and data self.key = "instance-id" self.key_data = "machine-uuid" @@ -264,34 +257,3 @@ class TestJujuUtils(ut_utils.BaseTestCase): key='series' ) self.assertEqual(expected, actual) - - def test_prepare_series_upgrade(self): - _machine_num = "1" - _to_series = "bionic" - juju_utils.prepare_series_upgrade(_machine_num, to_series=_to_series) - self.subprocess.check_call.assert_called_once_with( - ["juju", "upgrade-series", "-m", self.model_name, - "prepare", _machine_num, _to_series, "--agree"]) - - def test_complete_series_upgrade(self): - _machine_num = "1" - juju_utils.complete_series_upgrade(_machine_num) - self.subprocess.check_call.assert_called_once_with( - ["juju", "upgrade-series", "-m", self.model_name, - "complete", _machine_num]) - - def test_set_series(self): - _application = "application" - _to_series = "bionic" - juju_utils.set_series(_application, _to_series) - self.subprocess.check_call.assert_called_once_with( - ["juju", "set-series", "-m", self.model_name, - _application, _to_series]) - - def test_update_series(self): - _machine_num = "1" - _to_series = "bionic" - juju_utils.update_series(_machine_num, _to_series) - self.subprocess.check_call.assert_called_once_with( - ["juju", "update-series", "-m", self.model_name, - _machine_num, _to_series]) diff --git a/zaza/model.py b/zaza/model.py index 9670eb3..181b214 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -1183,3 +1183,62 @@ class UnitNotFound(Exception): msg = ('Unit: {} was not found in current model'. format(unit_name)) super(UnitNotFound, self).__init__(msg) + + +# NOTE: The following are series upgrade related functions which are new +# features in juju. We can migrate to libjuju calls when the feature +# stabilizes. +def prepare_series_upgrade(machine_num, to_series="xenial"): + """Execute juju series-upgrade prepare on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + + :param machine_num: Machine number + :type machine_num: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + "prepare", machine_num, to_series, "--agree"] + subprocess.check_call(cmd) + + +def complete_series_upgrade(machine_num): + """Execute juju series-upgrade complete on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + + :param machine_num: Machine number + :type machine_num: str + :returns: None + :rtype: None + """ + juju_model = get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + "complete", machine_num] + subprocess.check_call(cmd) + + +def set_series(application, to_series): + """Execute juju set-series complete on application. + + NOTE: This is a new feature in juju and not yet in libjuju. + + :param application: Name of application to upgrade series + :type application: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = get_juju_model() + cmd = ["juju", "set-series", "-m", juju_model, + application, to_series] + subprocess.check_call(cmd) diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index 74ddfe5..b5a549c 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -201,7 +201,7 @@ def series_upgrade_application(application, pause_non_leader_primary=True, :returns: None :rtype: None """ - status = juju_utils.get_application_status(application=application) + status = model.get_status().applications[application] # For some applications (percona-cluster) the leader unit must upgrade # first. For API applications the non-leader haclusters must be paused @@ -270,7 +270,7 @@ def series_upgrade(unit_name, machine_num, logging.info("Series upgrade {}".format(unit_name)) application = unit_name.split('/')[0] logging.info("Prepare series upgrade on {}".format(machine_num)) - juju_utils.prepare_series_upgrade(machine_num, to_series=to_series) + model.prepare_series_upgrade(machine_num, to_series=to_series) logging.info("Watiing for workload status 'unknown' on {}" .format(unit_name)) model.block_until_unit_wl_status(unit_name, "unknown") @@ -285,7 +285,7 @@ def series_upgrade(unit_name, machine_num, logging.info("Watiing for model idleness") model.block_until_all_units_idle() logging.info("Complete series upgrade on {}".format(machine_num)) - juju_utils.complete_series_upgrade(machine_num) + model.complete_series_upgrade(machine_num) model.block_until_all_units_idle() logging.info("Watiing for workload status 'active' on {}" .format(unit_name)) @@ -296,7 +296,7 @@ def series_upgrade(unit_name, machine_num, model.block_until_all_units_idle() # This step may be performed by juju in the future logging.info("Set series on {} to {}".format(application, to_series)) - juju_utils.set_series(application, to_series) + model.set_series(application, to_series) def set_origin(application, origin='openstack-origin', pocket='distro'): diff --git a/zaza/utilities/juju.py b/zaza/utilities/juju.py index 642a3d4..6f3c792 100644 --- a/zaza/utilities/juju.py +++ b/zaza/utilities/juju.py @@ -16,7 +16,6 @@ """Module for interacting with juju.""" import os from pathlib import Path -import subprocess import yaml from zaza import ( @@ -259,77 +258,3 @@ def leader_get(application, key=''): return yaml.load(result.get('Stdout')) else: raise model.CommandRunFailed(cmd, result) - - -def prepare_series_upgrade(machine_num, to_series="xenial"): - """Execute juju series-upgrade prepare on machine. - - NOTE: This is a new feature in juju behind a feature flag and not yet in - libjuju. - export JUJU_DEV_FEATURE_FLAGS=upgrade-series - - :param machine_num: Machine number - :type machine_num: str - :param to_series: The series to which to upgrade - :type to_series: str - :returns: None - :rtype: None - """ - juju_model = model.get_juju_model() - cmd = ["juju", "upgrade-series", "-m", juju_model, - "prepare", machine_num, to_series, "--agree"] - subprocess.check_call(cmd) - - -def complete_series_upgrade(machine_num): - """Execute juju series-upgrade complete on machine. - - NOTE: This is a new feature in juju behind a feature flag and not yet in - libjuju. - export JUJU_DEV_FEATURE_FLAGS=upgrade-series - - :param machine_num: Machine number - :type machine_num: str - :returns: None - :rtype: None - """ - juju_model = model.get_juju_model() - cmd = ["juju", "upgrade-series", "-m", juju_model, - "complete", machine_num] - subprocess.check_call(cmd) - - -def set_series(application, to_series): - """Execute juju set-series complete on application. - - NOTE: This is a new feature in juju and not yet in libjuju. - - :param application: Name of application to upgrade series - :type application: str - :param to_series: The series to which to upgrade - :type to_series: str - :returns: None - :rtype: None - """ - juju_model = model.get_juju_model() - cmd = ["juju", "set-series", "-m", juju_model, - application, to_series] - subprocess.check_call(cmd) - - -def update_series(machine_num, to_series): - """Execute juju update-series complete on machine. - - NOTE: This is a new feature in juju and not yet in libjuju. - - :param machine_num: Machine number - :type machine_num: str - :param to_series: The series to which to upgrade - :type to_series: str - :returns: None - :rtype: None - """ - juju_model = model.get_juju_model() - cmd = ["juju", "update-series", "-m", juju_model, - machine_num, to_series] - subprocess.check_call(cmd)