Use block_until_unit_wl_status for upgrade timing

Remove all calls to time.sleep and use block_until_unit_wl_status to
manage the timing of the upgrade steps.

Also use run_via_ssh to execute commands while the juju agents are down
during a series upgrade.

unit_tests/utilities/test_zaza_utilities_generic.py

@@ -168,8 +168,8 @@ class TestGenericUtils(ut_utils.BaseTestCase):
 
     def test_wrap_do_release_upgrade(self):
         self.patch_object(generic_utils, "do_release_upgrade")
+        self.patch_object(generic_utils, "run_via_ssh")
         self.patch_object(generic_utils.model, "scp_to_unit")
-        self.patch_object(generic_utils.model, "run_on_unit")
         _unit = "app/2"
         _from_series = "xenial"
         _to_series = "bionic"
@@ -177,26 +177,31 @@ class TestGenericUtils(ut_utils.BaseTestCase):
         _files = ["filename", _workaround_script]
         _scp_calls = []
         _run_calls = [
-            mock.call(_unit, _workaround_script),
-            mock.call(_unit, "juju-updateseries "
-                             "--from-series={} --to-series={}"
-                             .format(_from_series, _to_series))]
+            mock.call(_unit, _workaround_script)]
         for filename in _files:
             _scp_calls.append(mock.call(_unit, filename, filename))
         generic_utils.wrap_do_release_upgrade(
             _unit, to_series=_to_series, from_series=_from_series,
             workaround_script=_workaround_script, files=_files)
         self.scp_to_unit.assert_has_calls(_scp_calls)
-        self.run_on_unit.assert_has_calls(_run_calls)
+        self.run_via_ssh.assert_has_calls(_run_calls)
         self.do_release_upgrade.assert_called_once_with(_unit)
 
     def test_reboot(self):
         _unit = "app/2"
         generic_utils.reboot(_unit)
         self.subprocess.check_call.assert_called_once_with(
-            ['juju', 'run', "--unit", _unit,
+            ['juju', 'ssh', _unit,
              'sudo', 'reboot', '&&', 'exit'])
 
+    def test_run_via_ssh(self):
+        _unit = "app/2"
+        _cmd = "hostname"
+        generic_utils.run_via_ssh(_unit, _cmd)
+        self.subprocess.check_call.assert_called_once_with(
+            ['juju', 'ssh', _unit,
+             'sudo ' + _cmd])
+
     def test_set_origin(self):
         "application, origin='openstack-origin', pocket='distro'):"
         self.patch_object(generic_utils.model, "set_application_config")
@@ -208,12 +213,11 @@ class TestGenericUtils(ut_utils.BaseTestCase):
             _application, {_origin: _pocket})
 
     def test_series_upgrade(self):
-        self.patch_object(generic_utils.time, "sleep")
         self.patch_object(generic_utils.model, "block_until_all_units_idle")
+        self.patch_object(generic_utils.model, "block_until_unit_wl_status")
         self.patch_object(generic_utils.juju_utils, "prepare_series_upgrade")
         self.patch_object(generic_utils.juju_utils, "complete_series_upgrade")
         self.patch_object(generic_utils.juju_utils, "set_series")
-        self.patch_object(generic_utils.juju_utils, "update_series")
         self.patch_object(generic_utils, "set_origin")
         self.patch_object(generic_utils, "wrap_do_release_upgrade")
         self.patch_object(generic_utils, "reboot")
@@ -237,7 +241,6 @@ class TestGenericUtils(ut_utils.BaseTestCase):
             workaround_script=_workaround_script, files=_files)
         self.complete_series_upgrade.assert_called_once_with(_machine_num)
         self.set_series.assert_called_once_with(_application, _to_series)
-        self.update_series.assert_called_once_with(_machine_num, _to_series)
         self.set_origin.assert_called_once_with(_application, _origin)
         self.reboot.assert_called_once_with(_unit)
 

zaza/utilities/generic.py

@@ -17,7 +17,6 @@
 import logging
 import os
 import subprocess
-import time
 import yaml
 
 from zaza import model
@@ -267,24 +266,39 @@ def series_upgrade(unit_name, machine_num,
     :returns: None
     :rtype: None
     """
+    logging.info("Series upgrade {}".format(unit_name))
     application = unit_name.split('/')[0]
+    logging.info("Prepare series upgrade on {}".format(machine_num))
     juju_utils.prepare_series_upgrade(machine_num, to_series=to_series)
     logging.info("Watiing for model idleness")
     model.block_until_all_units_idle()
+    logging.info("Watiing for workload status 'unknown' on {}"
+                 .format(unit_name))
+    model.block_until_unit_wl_status(unit_name, "unknown")
     wrap_do_release_upgrade(unit_name, from_series=from_series,
                             to_series=to_series, files=files,
                             workaround_script=workaround_script)
-    reboot(unit_name)
-    # Without the sleep model.block_on_all_units_idle returns to early
-    logging.info("Sleeping after reboot...")
-    time.sleep(30)
     model.block_until_all_units_idle()
+    logging.info("Reboot {}".format(unit_name))
+    reboot(unit_name)
+    logging.info("Watiing for workload status 'blocked' on {}"
+                 .format(unit_name))
+    model.block_until_unit_wl_status(unit_name, "blocked")
+    logging.info("Watiing for model idleness")
+    model.block_until_all_units_idle()
+    logging.info("Complete series upgrade on {}".format(machine_num))
     juju_utils.complete_series_upgrade(machine_num)
     model.block_until_all_units_idle()
-    juju_utils.update_series(machine_num, to_series)
-    juju_utils.set_series(application, to_series)
+    logging.info("Watiing for workload status 'active' on {}"
+                 .format(unit_name))
+    model.block_until_unit_wl_status(unit_name, "active")
+    model.block_until_all_units_idle()
+    logging.info("Set origin on {}".format(application))
     set_origin(application, origin)
     model.block_until_all_units_idle()
+    # This step may be performed by juju in the future
+    logging.info("Set series on {} to {}".format(application, to_series))
+    juju_utils.set_series(application, to_series)
 
 
 def set_origin(application, origin='openstack-origin', pocket='distro'):
@@ -301,6 +315,7 @@ def set_origin(application, origin='openstack-origin', pocket='distro'):
     :returns: None
     :rtype: None
     """
+    logging.info("Set origin on {} to {}".format(application, origin))
     model.set_application_config(application, {origin: pocket})
 
 
@@ -330,22 +345,40 @@ def wrap_do_release_upgrade(unit_name, from_series="trusty",
     # to overcome some packaging bugs.
     # Copy scripts
     if files:
+        logging.info("SCP files")
         for _file in files:
+            logging.info("SCP {}".format(_file))
             model.scp_to_unit(unit_name, _file, os.path.basename(_file))
 
     # Run Scripts
     if workaround_script:
-        model.run_on_unit(unit_name, workaround_script)
+        logging.info("Running workaround script")
+        run_via_ssh(unit_name, workaround_script)
 
     # Actually do the do_release_upgrade
     do_release_upgrade(unit_name)
 
-    # Post upgrade hacks
-    # Juju may at some point in the future auotmate this step
-    model.run_on_unit(
-        unit_name,
-        "juju-updateseries --from-series={} --to-series={}"
-        .format(from_series, to_series))
+
+def run_via_ssh(unit_name, cmd):
+    """Run command on unit via ssh.
+
+    For executing commands on units when the juju agent is down.
+
+    :param unit_name: Unit Name
+    :param cmd: Command to execute on remote unit
+    :type cmd: str
+    :returns: None
+    :rtype: None
+    """
+    if "sudo" not in cmd:
+        cmd = "sudo {}".format(cmd)
+    cmd = ['juju', 'ssh', unit_name, cmd]
+    logging.info("Running {} on {}".format(cmd, unit_name))
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError as e:
+        logging.warn("Failed command {} on {}".format(cmd, unit_name))
+        logging.warn(e)
 
 
 def do_release_upgrade(unit_name):
@@ -376,8 +409,9 @@ def reboot(unit_name):
     :returns: None
     :rtype: None
     """
-    # NOTE: Runnig this via model.run_on_unit fails to exit properly
-    cmd = ['juju', 'run', '--unit', unit_name, 'sudo', 'reboot', '&&', 'exit']
+    # NOTE: When used with series upgrade the agent will be down.
+    # Even juju run will not work
+    cmd = ['juju', 'ssh', unit_name, 'sudo', 'reboot', '&&', 'exit']
     try:
         subprocess.check_call(cmd)
     except subprocess.CalledProcessError as e: