Merge pull request #118 from thedac/series-upgrade

Series upgrade

unit_tests/test_zaza_charm_lifecycle_prepare.py

@@ -75,7 +75,6 @@ class TestCharmLifecyclePrepare(ut_utils.BaseTestCase):
         self.add_model.assert_called_once_with(
             'newmodel',
             config={
-                'agent-stream': 'proposed',
                 'default-series': 'xenial',
                 'image-stream': 'daily',
                 'test-mode': 'true',

unit_tests/test_zaza_model.py

@@ -112,6 +112,25 @@ class TestModel(ut_utils.BaseTestCase):
         }
         self.Model_mock = mock.MagicMock()
 
+        # Juju Status Object and data
+        self.key = "instance-id"
+        self.key_data = "machine-uuid"
+        self.machine = "1"
+        self.machine_data = {self.key: self.key_data}
+        self.unit = "app/1"
+        self.unit_data = {
+            "workload-status": {"status": "active"},
+            "machine": self.machine}
+        self.application = "app"
+        self.application_data = {"units": {self.unit: self.unit_data}}
+        self.subordinate_application = "subordinate_application"
+        self.subordinate_application_data = {
+            "subordinate-to": [self.application]}
+        self.juju_status = mock.MagicMock()
+        self.juju_status.applications = {
+            self.application: self.application_data}
+        self.juju_status.machines = self.machine_data
+
         async def _connect_model(model_name):
             return model_name
 
@@ -886,34 +905,49 @@ disk_formats = ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar
 
     def test_block_until_unit_wl_status(self):
         async def _block_until(f, timeout=None):
-            if not f():
+            rc = await f()
+            if not rc:
                 raise asyncio.futures.TimeoutError
-        self.patch_object(model, 'get_juju_model', return_value='mname')
+
+        async def _get_status():
+            return self.juju_status
+
         self.patch_object(model, 'Model')
         self.Model.return_value = self.Model_mock
-        self.Model_mock.block_until.side_effect = _block_until
+        self.patch_object(model, 'get_juju_model', return_value='mname')
         self.patch_object(model, 'get_unit_from_name')
-        self.get_unit_from_name.return_value = mock.MagicMock(
-            workload_status='active')
+        self.patch_object(model, 'async_get_status')
+        self.async_get_status.side_effect = _get_status
+        self.patch_object(model, 'async_block_until')
+        self.async_block_until.side_effect = _block_until
         model.block_until_unit_wl_status(
-            'app/2',
+            'app/1',
             'active',
             timeout=0.1)
 
     def test_block_until_unit_wl_status_fail(self):
         async def _block_until(f, timeout=None):
-            if not f():
+            rc = await f()
+            if not rc:
                 raise asyncio.futures.TimeoutError
-        self.patch_object(model, 'get_juju_model', return_value='mname')
+
+        async def _get_status():
+            return self.juju_status
+
+        (self.juju_status.applications[self.application]
+            ["units"][self.unit]["workload-status"]["status"]) = "blocked"
+
         self.patch_object(model, 'Model')
         self.Model.return_value = self.Model_mock
-        self.Model_mock.block_until.side_effect = _block_until
+        self.patch_object(model, 'get_juju_model', return_value='mname')
         self.patch_object(model, 'get_unit_from_name')
-        self.get_unit_from_name.return_value = mock.MagicMock(
-            workload_status='maintenance')
+        self.patch_object(model, 'async_get_status')
+        self.async_get_status.side_effect = _get_status
+        self.patch_object(model, 'async_block_until')
+        self.async_block_until.side_effect = _block_until
         with self.assertRaises(asyncio.futures.TimeoutError):
             model.block_until_unit_wl_status(
-                'app/2',
+                'app/1',
                 'active',
                 timeout=0.1)
 
@@ -940,6 +974,38 @@ disk_formats = ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar
         with self.assertRaises(asyncio.futures.TimeoutError):
             model.wait_for_agent_status(timeout=0.1)
 
+    def test_prepare_series_upgrade(self):
+        self.patch_object(model, 'subprocess')
+        self.patch_object(model, 'get_juju_model',
+                          return_value=self.model_name)
+        _machine_num = "1"
+        _to_series = "bionic"
+        model.prepare_series_upgrade(_machine_num, to_series=_to_series)
+        self.subprocess.check_call.assert_called_once_with(
+            ["juju", "upgrade-series", "-m", self.model_name,
+             "prepare", _machine_num, _to_series, "--agree"])
+
+    def test_complete_series_upgrade(self):
+        self.patch_object(model, 'get_juju_model',
+                          return_value=self.model_name)
+        self.patch_object(model, 'subprocess')
+        _machine_num = "1"
+        model.complete_series_upgrade(_machine_num)
+        self.subprocess.check_call.assert_called_once_with(
+            ["juju", "upgrade-series", "-m", self.model_name,
+             "complete", _machine_num])
+
+    def test_set_series(self):
+        self.patch_object(model, 'get_juju_model',
+                          return_value=self.model_name)
+        self.patch_object(model, 'subprocess')
+        _application = "application"
+        _to_series = "bionic"
+        model.set_series(_application, _to_series)
+        self.subprocess.check_call.assert_called_once_with(
+            ["juju", "set-series", "-m", self.model_name,
+             _application, _to_series])
+
 
 class AsyncModelTests(aiounittest.AsyncTestCase):
 

unit_tests/utilities/test_zaza_utilities_generic.py

@@ -16,11 +16,42 @@ import mock
 import unit_tests.utils as ut_utils
 from zaza.utilities import generic as generic_utils
 
+FAKE_STATUS = {
+    'can-upgrade-to': '',
+    'charm': 'local:trusty/app-136',
+    'subordinate-to': [],
+    'units': {'app/0': {'leader': True,
+                        'machine': '0',
+                        'subordinates': {
+                            'app-hacluster/0': {
+                                   'charm': 'local:trusty/hacluster-0',
+                                   'leader': True}}},
+              'app/1': {'machine': '1',
+                        'subordinates': {
+                            'app-hacluster/1': {
+                                   'charm': 'local:trusty/hacluster-0'}}},
+              'app/2': {'machine': '2',
+                        'subordinates': {
+                            'app-hacluster/2': {
+                                   'charm': 'local:trusty/hacluster-0'}}}}}
+
 
 class TestGenericUtils(ut_utils.BaseTestCase):
 
     def setUp(self):
         super(TestGenericUtils, self).setUp()
+        # Patch all subprocess calls
+        self.patch(
+            'zaza.utilities.generic.subprocess',
+            new_callable=mock.MagicMock(),
+            name='subprocess'
+        )
+
+        # Juju Status Object and data
+        self.juju_status = mock.MagicMock()
+        self.juju_status.applications.__getitem__.return_value = FAKE_STATUS
+        self.patch_object(generic_utils, "model")
+        self.model.get_status.return_value = self.juju_status
 
     def test_dict_to_yaml(self):
         _dict_data = {"key": "value"}
@@ -133,3 +164,190 @@ class TestGenericUtils(ut_utils.BaseTestCase):
         self.assertEqual(generic_utils.get_yaml_config(_filename),
                          _yaml_dict)
         self._open.assert_called_once_with(_filename, "r")
+
+    def test_do_release_upgrade(self):
+        _unit = "app/2"
+        generic_utils.do_release_upgrade(_unit)
+        self.subprocess.check_call.assert_called_once_with(
+            ['juju', 'ssh', _unit, 'sudo', 'do-release-upgrade',
+             '-f', 'DistUpgradeViewNonInteractive'])
+
+    def test_wrap_do_release_upgrade(self):
+        self.patch_object(generic_utils, "do_release_upgrade")
+        self.patch_object(generic_utils, "run_via_ssh")
+        self.patch_object(generic_utils.model, "scp_to_unit")
+        _unit = "app/2"
+        _from_series = "xenial"
+        _to_series = "bionic"
+        _workaround_script = "scriptname"
+        _files = ["filename", _workaround_script]
+        _scp_calls = []
+        _run_calls = [
+            mock.call(_unit, _workaround_script)]
+        for filename in _files:
+            _scp_calls.append(mock.call(_unit, filename, filename))
+        generic_utils.wrap_do_release_upgrade(
+            _unit, to_series=_to_series, from_series=_from_series,
+            workaround_script=_workaround_script, files=_files)
+        self.scp_to_unit.assert_has_calls(_scp_calls)
+        self.run_via_ssh.assert_has_calls(_run_calls)
+        self.do_release_upgrade.assert_called_once_with(_unit)
+
+    def test_reboot(self):
+        _unit = "app/2"
+        generic_utils.reboot(_unit)
+        self.subprocess.check_call.assert_called_once_with(
+            ['juju', 'ssh', _unit,
+             'sudo', 'reboot', '&&', 'exit'])
+
+    def test_run_via_ssh(self):
+        _unit = "app/2"
+        _cmd = "hostname"
+        generic_utils.run_via_ssh(_unit, _cmd)
+        self.subprocess.check_call.assert_called_once_with(
+            ['juju', 'ssh', _unit,
+             'sudo ' + _cmd])
+
+    def test_set_origin(self):
+        "application, origin='openstack-origin', pocket='distro'):"
+        self.patch_object(generic_utils.model, "set_application_config")
+        _application = "application"
+        _origin = "source"
+        _pocket = "cloud:fake-cloud"
+        generic_utils.set_origin(_application, origin=_origin, pocket=_pocket)
+        self.set_application_config.assert_called_once_with(
+            _application, {_origin: _pocket})
+
+    def test_series_upgrade(self):
+        self.patch_object(generic_utils.model, "block_until_all_units_idle")
+        self.patch_object(generic_utils.model, "block_until_unit_wl_status")
+        self.patch_object(generic_utils.model, "prepare_series_upgrade")
+        self.patch_object(generic_utils.model, "complete_series_upgrade")
+        self.patch_object(generic_utils.model, "set_series")
+        self.patch_object(generic_utils, "set_origin")
+        self.patch_object(generic_utils, "wrap_do_release_upgrade")
+        self.patch_object(generic_utils, "reboot")
+        _unit = "app/2"
+        _application = "app"
+        _machine_num = "4"
+        _from_series = "xenial"
+        _to_series = "bionic"
+        _origin = "source"
+        _files = ["filename", "scriptname"]
+        _workaround_script = "scriptname"
+        generic_utils.series_upgrade(
+            _unit, _machine_num, origin=_origin,
+            to_series=_to_series, from_series=_from_series,
+            workaround_script=_workaround_script, files=_files)
+        self.block_until_all_units_idle.called_with()
+        self.prepare_series_upgrade.assert_called_once_with(
+            _machine_num, to_series=_to_series)
+        self.wrap_do_release_upgrade.assert_called_once_with(
+            _unit, to_series=_to_series, from_series=_from_series,
+            workaround_script=_workaround_script, files=_files)
+        self.complete_series_upgrade.assert_called_once_with(_machine_num)
+        self.set_series.assert_called_once_with(_application, _to_series)
+        self.set_origin.assert_called_once_with(_application, _origin)
+        self.reboot.assert_called_once_with(_unit)
+
+    def test_series_upgrade_application_pause_peers_and_subordinates(self):
+        self.patch_object(generic_utils.model, "run_action")
+        self.patch_object(generic_utils, "series_upgrade")
+        _application = "app"
+        _from_series = "xenial"
+        _to_series = "bionic"
+        _origin = "source"
+        _files = ["filename", "scriptname"]
+        _workaround_script = "scriptname"
+        # Peers and Subordinates
+        _run_action_calls = [
+            mock.call("{}-hacluster/1".format(_application),
+                      "pause", action_params={}),
+            mock.call("{}/1".format(_application), "pause", action_params={}),
+            mock.call("{}-hacluster/2".format(_application),
+                      "pause", action_params={}),
+            mock.call("{}/2".format(_application), "pause", action_params={}),
+        ]
+        _series_upgrade_calls = []
+        for machine_num in ("0", "1", "2"):
+            _series_upgrade_calls.append(
+                mock.call("{}/{}".format(_application, machine_num),
+                          machine_num, origin=_origin,
+                          from_series=_from_series, to_series=_to_series,
+                          workaround_script=_workaround_script, files=_files),
+            )
+
+        # Pause primary peers and subordinates
+        generic_utils.series_upgrade_application(
+            _application, origin=_origin,
+            to_series=_to_series, from_series=_from_series,
+            pause_non_leader_primary=True,
+            pause_non_leader_subordinate=True,
+            workaround_script=_workaround_script, files=_files),
+        self.run_action.assert_has_calls(_run_action_calls)
+        self.series_upgrade.assert_has_calls(_series_upgrade_calls)
+
+    def test_series_upgrade_application_pause_subordinates(self):
+        self.patch_object(generic_utils.model, "run_action")
+        self.patch_object(generic_utils, "series_upgrade")
+        _application = "app"
+        _from_series = "xenial"
+        _to_series = "bionic"
+        _origin = "source"
+        _files = ["filename", "scriptname"]
+        _workaround_script = "scriptname"
+        # Subordinates only
+        _run_action_calls = [
+            mock.call("{}-hacluster/1".format(_application),
+                      "pause", action_params={}),
+            mock.call("{}-hacluster/2".format(_application),
+                      "pause", action_params={}),
+        ]
+        _series_upgrade_calls = []
+
+        for machine_num in ("0", "1", "2"):
+            _series_upgrade_calls.append(
+                mock.call("{}/{}".format(_application, machine_num),
+                          machine_num, origin=_origin,
+                          from_series=_from_series, to_series=_to_series,
+                          workaround_script=_workaround_script, files=_files),
+            )
+
+        # Pause subordinates
+        generic_utils.series_upgrade_application(
+            _application, origin=_origin,
+            to_series=_to_series, from_series=_from_series,
+            pause_non_leader_primary=False,
+            pause_non_leader_subordinate=True,
+            workaround_script=_workaround_script, files=_files),
+        self.run_action.assert_has_calls(_run_action_calls)
+        self.series_upgrade.assert_has_calls(_series_upgrade_calls)
+
+    def test_series_upgrade_application_no_pause(self):
+        self.patch_object(generic_utils.model, "run_action")
+        self.patch_object(generic_utils, "series_upgrade")
+        _application = "app"
+        _from_series = "xenial"
+        _to_series = "bionic"
+        _origin = "source"
+        _series_upgrade_calls = []
+        _files = ["filename", "scriptname"]
+        _workaround_script = "scriptname"
+
+        for machine_num in ("0", "1", "2"):
+            _series_upgrade_calls.append(
+                mock.call("{}/{}".format(_application, machine_num),
+                          machine_num, origin=_origin,
+                          from_series=_from_series, to_series=_to_series,
+                          workaround_script=_workaround_script, files=_files),
+            )
+
+        # No Pausiing
+        generic_utils.series_upgrade_application(
+            _application, origin=_origin,
+            to_series=_to_series, from_series=_from_series,
+            pause_non_leader_primary=False,
+            pause_non_leader_subordinate=False,
+            workaround_script=_workaround_script, files=_files)
+        self.run_action.assert_not_called()
+        self.series_upgrade.assert_has_calls(_series_upgrade_calls)

unit_tests/utilities/test_zaza_utilities_juju.py

@@ -42,6 +42,7 @@ class TestJujuUtils(ut_utils.BaseTestCase):
         # Model
         self.patch_object(juju_utils, "model")
         self.model_name = "model-name"
+        self.model.get_juju_model.return_value = self.model_name
         self.model.get_status.return_value = self.juju_status
         self.run_output = {"Code": "0", "Stderr": "", "Stdout": "RESULT"}
         self.error_run_output = {"Code": "1", "Stderr": "ERROR", "Stdout": ""}

zaza/charm_lifecycle/prepare.py

@@ -25,7 +25,6 @@ import zaza.model
 MODEL_DEFAULTS = {
     # Model defaults from charm-test-infra
     #   https://jujucharms.com/docs/2.1/models-config
-    'agent-stream': 'proposed',
     'default-series': 'xenial',
     'image-stream': 'daily',
     'test-mode': 'true',

zaza/charm_tests/nova/tests.py

@@ -112,7 +112,7 @@ class BaseGuestCreateTest(unittest.TestCase):
 class CirrosGuestCreateTest(BaseGuestCreateTest):
     """Tests to launch a cirros image."""
 
-    def test_launch_small_cirros_instance(self):
+    def test_launch_small_instance(self):
         """Launch a cirros instance and test connectivity."""
         self.launch_instance(glance_setup.CIRROS_IMAGE_NAME)
 
@@ -120,6 +120,6 @@ class CirrosGuestCreateTest(BaseGuestCreateTest):
 class LTSGuestCreateTest(BaseGuestCreateTest):
     """Tests to launch a LTS image."""
 
-    def test_launch_small_cirros_instance(self):
-        """Launch a cirros instance and test connectivity."""
+    def test_launch_small_instance(self):
+        """Launch a Bionic instance and test connectivity."""
         self.launch_instance(glance_setup.LTS_IMAGE_NAME)

zaza/charm_tests/series_upgrade/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2018 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test series upgrade."""

zaza/charm_tests/series_upgrade/tests.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+# Copyright 2018 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Define class for Series Upgrade."""
+
+import logging
+import os
+import unittest
+
+from zaza import model
+from zaza.utilities import (
+    cli as cli_utils,
+    generic as generic_utils,
+)
+from zaza.charm_tests.nova.tests import LTSGuestCreateTest
+
+
+class SeriesUpgradeTest(unittest.TestCase):
+    """Class to encapsulate Sereis Upgrade Tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run setup for Series Upgrades."""
+        cli_utils.setup_logging()
+        cls.lts = LTSGuestCreateTest()
+
+    def validate_pre_series_upgrade_cloud(self):
+        """Validate pre series upgrade."""
+        logging.info("Validate pre-series-upgrade: Spin up LTS instance")
+        self.lts.test_launch_small_instance()
+
+    def test_200_run_series_upgrade(self):
+        """Run series upgrade."""
+        # Set Feature Flag
+        os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
+
+        # While there are packaging upgrade bugs we need to be cheeky and
+        # workaround by using the new package's version of files
+        workaround_script = "/home/ubuntu/package-workarounds.sh"
+        src_workaround_script = os.path.basename(workaround_script)
+
+        files = [src_workaround_script, 'corosync', 'corosync.conf']
+
+        applications = model.get_status().applications
+        from_series = "trusty"
+        to_series = "xenial"
+        for application in applications:
+            # Defaults
+            origin = "openstack-origin"
+            pause_non_leader_subordinate = True
+            pause_non_leader_primary = False
+            # Skip subordinates
+            if applications[application]["subordinate-to"]:
+                continue
+            if "percona-cluster" in applications[application]["charm"]:
+                origin = "source"
+                pause_non_leader_primary = True
+                pause_non_leader_subordinate = True
+            if "rabbitmq-server" in applications[application]["charm"]:
+                origin = "source"
+                pause_non_leader_primary = True
+                pause_non_leader_subordinate = False
+            if "nova-compute" in applications[application]["charm"]:
+                pause_non_leader_primary = False
+                pause_non_leader_subordinate = False
+            # Place holder for Ceph applications
+            # The rest are likley APIs and use defaults
+
+            generic_utils.series_upgrade_application(
+                application,
+                pause_non_leader_primary=pause_non_leader_primary,
+                pause_non_leader_subordinate=pause_non_leader_subordinate,
+                from_series=from_series,
+                to_series=to_series,
+                origin=origin,
+                workaround_script=workaround_script,
+                files=files)
+
+    def validate_series_upgraded_cloud(self):
+        """Validate post series upgrade."""
+        logging.info("Validate post-series-upgrade: Spin up LTS instance")
+        self.lts.test_launch_small_instance()
+
+
+if __name__ == "__main__":
+    unittest.main()

zaza/model.py

@@ -1090,25 +1090,31 @@ async def async_block_until_unit_wl_status(unit_name, status, model_name=None,
     blocks until the given unit has the desired workload status::
 
          block_until_unit_wl_status(
-            'modelname',
             aunit,
-            'active')
+            'active'
+            model_name='modelname')
 
-    :param model_name: Name of model to query.
-    :type model_name: str
-    :param unit_name: Name of unit to run action on
+    NOTE: unit.workload_status was actually reporting the application workload
+          status. Using the full status output from model.get_status() gives us
+          unit by unit workload status.
+
+    :param unit_name: Name of unit
     :type unit_name: str
     :param status: Status to wait for (active, maintenance etc)
     :type status: str
+    :param model_name: Name of model to query.
+    :type model_name: str
     :param timeout: Time to wait for unit to achieved desired status
     :type timeout: float
     """
-    async with run_in_model(model_name) as model:
-        unit = get_unit_from_name(unit_name, model)
-        await model.block_until(
-            lambda: unit.workload_status == status,
-            timeout=timeout
-        )
+    async def _unit_status():
+        app = unit_name.split("/")[0]
+        model_status = await async_get_status()
+        return (model_status.applications[app]['units'][unit_name]
+                ['workload-status']['status'] == status)
+
+    async with run_in_model(model_name):
+        await async_block_until(_unit_status, timeout=timeout)
 
 block_until_unit_wl_status = sync_wrapper(
     async_block_until_unit_wl_status)
@@ -1177,3 +1183,62 @@ class UnitNotFound(Exception):
         msg = ('Unit: {} was not found in current model'.
                format(unit_name))
         super(UnitNotFound, self).__init__(msg)
+
+
+# NOTE: The following are series upgrade related functions which are new
+# features in juju. We can migrate to libjuju calls when the feature
+# stabilizes.
+def prepare_series_upgrade(machine_num, to_series="xenial"):
+    """Execute juju series-upgrade prepare on machine.
+
+    NOTE: This is a new feature in juju behind a feature flag and not yet in
+    libjuju.
+    export JUJU_DEV_FEATURE_FLAGS=upgrade-series
+
+    :param machine_num: Machine number
+    :type machine_num: str
+    :param to_series: The series to which to upgrade
+    :type to_series: str
+    :returns: None
+    :rtype: None
+    """
+    juju_model = get_juju_model()
+    cmd = ["juju", "upgrade-series", "-m", juju_model,
+           "prepare", machine_num, to_series, "--agree"]
+    subprocess.check_call(cmd)
+
+
+def complete_series_upgrade(machine_num):
+    """Execute juju series-upgrade complete on machine.
+
+    NOTE: This is a new feature in juju behind a feature flag and not yet in
+    libjuju.
+    export JUJU_DEV_FEATURE_FLAGS=upgrade-series
+
+    :param machine_num: Machine number
+    :type machine_num: str
+    :returns: None
+    :rtype: None
+    """
+    juju_model = get_juju_model()
+    cmd = ["juju", "upgrade-series", "-m", juju_model,
+           "complete", machine_num]
+    subprocess.check_call(cmd)
+
+
+def set_series(application, to_series):
+    """Execute juju set-series complete on application.
+
+    NOTE: This is a new feature in juju and not yet in libjuju.
+
+    :param application: Name of application to upgrade series
+    :type application: str
+    :param to_series: The series to which to upgrade
+    :type to_series: str
+    :returns: None
+    :rtype: None
+    """
+    juju_model = get_juju_model()
+    cmd = ["juju", "set-series", "-m", juju_model,
+           application, to_series]
+    subprocess.check_call(cmd)

zaza/utilities/generic.py

@@ -16,6 +16,7 @@
 
 import logging
 import os
+import subprocess
 import yaml
 
 from zaza import model
@@ -165,3 +166,252 @@ def get_yaml_config(config_file):
     # the pwd.
     logging.info('Using config %s' % (config_file))
     return yaml.load(open(config_file, 'r').read())
+
+
+def series_upgrade_application(application, pause_non_leader_primary=True,
+                               pause_non_leader_subordinate=True,
+                               from_series="trusty", to_series="xenial",
+                               origin='openstack-origin',
+                               files=None, workaround_script=None):
+    """Series upgrade application.
+
+    Wrap all the functionality to handle series upgrade for a given
+    application. Including pausing non-leader units.
+
+    :param application: Name of application to upgrade series
+    :type application: str
+    :param pause_non_leader_primary: Whether the non-leader applications should
+                                     be paused
+    :type pause_non_leader_primary: bool
+    :param pause_non_leader_subordinate: Whether the non-leader subordinate
+                                         hacluster applications should be
+                                         paused
+    :type pause_non_leader_subordinate: bool
+    :param from_series: The series from which to upgrade
+    :type from_series: str
+    :param to_series: The series to which to upgrade
+    :type to_series: str
+    :param origin: The configuration setting variable name for changing origin
+                   source. (openstack-origin or source)
+    :type origin: str
+    :param files: Workaround files to scp to unit under upgrade
+    :type files: list
+    :param workaround_script: Workaround script to run during series upgrade
+    :type workaround_script: str
+    :returns: None
+    :rtype: None
+    """
+    status = model.get_status().applications[application]
+
+    # For some applications (percona-cluster) the leader unit must upgrade
+    # first. For API applications the non-leader haclusters must be paused
+    # before upgrade. Finally, for some applications this is aribtrary but
+    # generalized.
+    leader = None
+    non_leaders = []
+    for unit in status["units"]:
+        if status["units"][unit].get("leader"):
+            leader = unit
+        else:
+            non_leaders.append(unit)
+
+    # Pause the non-leaders
+    for unit in non_leaders:
+        if pause_non_leader_subordinate:
+            if status["units"][unit].get("subordinates"):
+                for subordinate in status["units"][unit]["subordinates"]:
+                    logging.info("Pausing {}".format(subordinate))
+                    model.run_action(subordinate, "pause", action_params={})
+        if pause_non_leader_primary:
+            logging.info("Pausing {}".format(unit))
+            model.run_action(unit, "pause", action_params={})
+
+    # Series upgrade the leader
+    logging.info("Series upgrade leader: {}".format(leader))
+    series_upgrade(leader, status["units"][leader]["machine"],
+                   from_series=from_series, to_series=to_series,
+                   origin=origin, workaround_script=workaround_script,
+                   files=files)
+
+    # Series upgrade the non-leaders
+    for unit in non_leaders:
+        logging.info("Series upgrade non-leader unit: {}"
+                     .format(unit))
+        series_upgrade(unit, status["units"][unit]["machine"],
+                       from_series=from_series, to_series=to_series,
+                       origin=origin, workaround_script=workaround_script,
+                       files=files)
+
+
+def series_upgrade(unit_name, machine_num,
+                   from_series="trusty", to_series="xenial",
+                   origin='openstack-origin',
+                   files=None, workaround_script=None):
+    """Perform series upgrade on a unit.
+
+    :param unit_name: Unit Name
+    :type unit_name: str
+    :param machine_num: Machine number
+    :type machine_num: str
+    :param from_series: The series from which to upgrade
+    :type from_series: str
+    :param to_series: The series to which to upgrade
+    :type to_series: str
+    :param origin: The configuration setting variable name for changing origin
+                   source. (openstack-origin or source)
+    :type origin: str
+    :param files: Workaround files to scp to unit under upgrade
+    :type files: list
+    :param workaround_script: Workaround script to run during series upgrade
+    :type workaround_script: str
+    :returns: None
+    :rtype: None
+    """
+    logging.info("Series upgrade {}".format(unit_name))
+    application = unit_name.split('/')[0]
+    logging.info("Prepare series upgrade on {}".format(machine_num))
+    model.prepare_series_upgrade(machine_num, to_series=to_series)
+    logging.info("Watiing for workload status 'unknown' on {}"
+                 .format(unit_name))
+    model.block_until_unit_wl_status(unit_name, "unknown")
+    wrap_do_release_upgrade(unit_name, from_series=from_series,
+                            to_series=to_series, files=files,
+                            workaround_script=workaround_script)
+    logging.info("Reboot {}".format(unit_name))
+    reboot(unit_name)
+    logging.info("Watiing for workload status 'blocked' on {}"
+                 .format(unit_name))
+    model.block_until_unit_wl_status(unit_name, "blocked")
+    logging.info("Watiing for model idleness")
+    model.block_until_all_units_idle()
+    logging.info("Complete series upgrade on {}".format(machine_num))
+    model.complete_series_upgrade(machine_num)
+    model.block_until_all_units_idle()
+    logging.info("Watiing for workload status 'active' on {}"
+                 .format(unit_name))
+    model.block_until_unit_wl_status(unit_name, "active")
+    model.block_until_all_units_idle()
+    logging.info("Set origin on {}".format(application))
+    set_origin(application, origin)
+    model.block_until_all_units_idle()
+    # This step may be performed by juju in the future
+    logging.info("Set series on {} to {}".format(application, to_series))
+    model.set_series(application, to_series)
+
+
+def set_origin(application, origin='openstack-origin', pocket='distro'):
+    """Set the configuration option for origin source.
+
+    :param application: Name of application to upgrade series
+    :type application: str
+    :param origin: The configuration setting variable name for changing origin
+                   source. (openstack-origin or source)
+    :type origin: str
+    :param pocket: Origin source cloud pocket.
+                   i.e. 'distro' or 'cloud:xenial-newton'
+    :type pocket: str
+    :returns: None
+    :rtype: None
+    """
+    logging.info("Set origin on {} to {}".format(application, origin))
+    model.set_application_config(application, {origin: pocket})
+
+
+def wrap_do_release_upgrade(unit_name, from_series="trusty",
+                            to_series="xenial",
+                            files=None, workaround_script=None):
+    """Wrap do release upgrade.
+
+    In a production environment this step would be run administratively.
+    For testing purposes we need this automated.
+
+    :param unit_name: Unit Name
+    :type unit_name: str
+    :param from_series: The series from which to upgrade
+    :type from_series: str
+    :param to_series: The series to which to upgrade
+    :type to_series: str
+    :param files: Workaround files to scp to unit under upgrade
+    :type files: list
+    :param workaround_script: Workaround script to run during series upgrade
+    :type workaround_script: str
+    :returns: None
+    :rtype: None
+    """
+    # Pre upgrade hacks
+    # There are a few necessary hacks to accomplish an automated upgrade
+    # to overcome some packaging bugs.
+    # Copy scripts
+    if files:
+        logging.info("SCP files")
+        for _file in files:
+            logging.info("SCP {}".format(_file))
+            model.scp_to_unit(unit_name, _file, os.path.basename(_file))
+
+    # Run Script
+    if workaround_script:
+        logging.info("Running workaround script")
+        run_via_ssh(unit_name, workaround_script)
+
+    # Actually do the do_release_upgrade
+    do_release_upgrade(unit_name)
+
+
+def run_via_ssh(unit_name, cmd):
+    """Run command on unit via ssh.
+
+    For executing commands on units when the juju agent is down.
+
+    :param unit_name: Unit Name
+    :param cmd: Command to execute on remote unit
+    :type cmd: str
+    :returns: None
+    :rtype: None
+    """
+    if "sudo" not in cmd:
+        cmd = "sudo {}".format(cmd)
+    cmd = ['juju', 'ssh', unit_name, cmd]
+    logging.info("Running {} on {}".format(cmd, unit_name))
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError as e:
+        logging.warn("Failed command {} on {}".format(cmd, unit_name))
+        logging.warn(e)
+
+
+def do_release_upgrade(unit_name):
+    """Run do-release-upgrade noninteractive.
+
+    :param unit_name: Unit Name
+    :type unit_name: str
+    :returns: None
+    :rtype: None
+    """
+    logging.info('Upgrading ' + unit_name)
+    # NOTE: It is necessary to run this via juju ssh rather than juju run due
+    # to timeout restrictions and error handling.
+    cmd = ['juju', 'ssh', unit_name, 'sudo',
+           'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive']
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError as e:
+        logging.warn("Failed do-release-upgrade for {}".format(unit_name))
+        logging.warn(e)
+
+
+def reboot(unit_name):
+    """Reboot unit.
+
+    :param unit_name: Unit Name
+    :type unit_name: str
+    :returns: None
+    :rtype: None
+    """
+    # NOTE: When used with series upgrade the agent will be down.
+    # Even juju run will not work
+    cmd = ['juju', 'ssh', unit_name, 'sudo', 'reboot', '&&', 'exit']
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError as e:
+        logging.info(e)
+        pass