diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index dc61736..76481fc 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -168,8 +168,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): def test_wrap_do_release_upgrade(self): self.patch_object(generic_utils, "do_release_upgrade") + self.patch_object(generic_utils, "run_via_ssh") self.patch_object(generic_utils.model, "scp_to_unit") - self.patch_object(generic_utils.model, "run_on_unit") _unit = "app/2" _from_series = "xenial" _to_series = "bionic" @@ -177,26 +177,31 @@ class TestGenericUtils(ut_utils.BaseTestCase): _files = ["filename", _workaround_script] _scp_calls = [] _run_calls = [ - mock.call(_unit, _workaround_script), - mock.call(_unit, "juju-updateseries " - "--from-series={} --to-series={}" - .format(_from_series, _to_series))] + mock.call(_unit, _workaround_script)] for filename in _files: _scp_calls.append(mock.call(_unit, filename, filename)) generic_utils.wrap_do_release_upgrade( _unit, to_series=_to_series, from_series=_from_series, workaround_script=_workaround_script, files=_files) self.scp_to_unit.assert_has_calls(_scp_calls) - self.run_on_unit.assert_has_calls(_run_calls) + self.run_via_ssh.assert_has_calls(_run_calls) self.do_release_upgrade.assert_called_once_with(_unit) def test_reboot(self): _unit = "app/2" generic_utils.reboot(_unit) self.subprocess.check_call.assert_called_once_with( - ['juju', 'run', "--unit", _unit, + ['juju', 'ssh', _unit, 'sudo', 'reboot', '&&', 'exit']) + def test_run_via_ssh(self): + _unit = "app/2" + _cmd = "hostname" + generic_utils.run_via_ssh(_unit, _cmd) + self.subprocess.check_call.assert_called_once_with( + ['juju', 'ssh', _unit, + 'sudo ' + _cmd]) + def test_set_origin(self): "application, origin='openstack-origin', pocket='distro'):" self.patch_object(generic_utils.model, "set_application_config") @@ -208,12 +213,11 @@ class TestGenericUtils(ut_utils.BaseTestCase): _application, {_origin: _pocket}) def test_series_upgrade(self): - self.patch_object(generic_utils.time, "sleep") self.patch_object(generic_utils.model, "block_until_all_units_idle") + self.patch_object(generic_utils.model, "block_until_unit_wl_status") self.patch_object(generic_utils.juju_utils, "prepare_series_upgrade") self.patch_object(generic_utils.juju_utils, "complete_series_upgrade") self.patch_object(generic_utils.juju_utils, "set_series") - self.patch_object(generic_utils.juju_utils, "update_series") self.patch_object(generic_utils, "set_origin") self.patch_object(generic_utils, "wrap_do_release_upgrade") self.patch_object(generic_utils, "reboot") @@ -237,7 +241,6 @@ class TestGenericUtils(ut_utils.BaseTestCase): workaround_script=_workaround_script, files=_files) self.complete_series_upgrade.assert_called_once_with(_machine_num) self.set_series.assert_called_once_with(_application, _to_series) - self.update_series.assert_called_once_with(_machine_num, _to_series) self.set_origin.assert_called_once_with(_application, _origin) self.reboot.assert_called_once_with(_unit) diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index 01502ef..b001db5 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -17,7 +17,6 @@ import logging import os import subprocess -import time import yaml from zaza import model @@ -267,24 +266,39 @@ def series_upgrade(unit_name, machine_num, :returns: None :rtype: None """ + logging.info("Series upgrade {}".format(unit_name)) application = unit_name.split('/')[0] + logging.info("Prepare series upgrade on {}".format(machine_num)) juju_utils.prepare_series_upgrade(machine_num, to_series=to_series) logging.info("Watiing for model idleness") model.block_until_all_units_idle() + logging.info("Watiing for workload status 'unknown' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "unknown") wrap_do_release_upgrade(unit_name, from_series=from_series, to_series=to_series, files=files, workaround_script=workaround_script) - reboot(unit_name) - # Without the sleep model.block_on_all_units_idle returns to early - logging.info("Sleeping after reboot...") - time.sleep(30) model.block_until_all_units_idle() + logging.info("Reboot {}".format(unit_name)) + reboot(unit_name) + logging.info("Watiing for workload status 'blocked' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "blocked") + logging.info("Watiing for model idleness") + model.block_until_all_units_idle() + logging.info("Complete series upgrade on {}".format(machine_num)) juju_utils.complete_series_upgrade(machine_num) model.block_until_all_units_idle() - juju_utils.update_series(machine_num, to_series) - juju_utils.set_series(application, to_series) + logging.info("Watiing for workload status 'active' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "active") + model.block_until_all_units_idle() + logging.info("Set origin on {}".format(application)) set_origin(application, origin) model.block_until_all_units_idle() + # This step may be performed by juju in the future + logging.info("Set series on {} to {}".format(application, to_series)) + juju_utils.set_series(application, to_series) def set_origin(application, origin='openstack-origin', pocket='distro'): @@ -301,6 +315,7 @@ def set_origin(application, origin='openstack-origin', pocket='distro'): :returns: None :rtype: None """ + logging.info("Set origin on {} to {}".format(application, origin)) model.set_application_config(application, {origin: pocket}) @@ -330,22 +345,40 @@ def wrap_do_release_upgrade(unit_name, from_series="trusty", # to overcome some packaging bugs. # Copy scripts if files: + logging.info("SCP files") for _file in files: + logging.info("SCP {}".format(_file)) model.scp_to_unit(unit_name, _file, os.path.basename(_file)) # Run Scripts if workaround_script: - model.run_on_unit(unit_name, workaround_script) + logging.info("Running workaround script") + run_via_ssh(unit_name, workaround_script) # Actually do the do_release_upgrade do_release_upgrade(unit_name) - # Post upgrade hacks - # Juju may at some point in the future auotmate this step - model.run_on_unit( - unit_name, - "juju-updateseries --from-series={} --to-series={}" - .format(from_series, to_series)) + +def run_via_ssh(unit_name, cmd): + """Run command on unit via ssh. + + For executing commands on units when the juju agent is down. + + :param unit_name: Unit Name + :param cmd: Command to execute on remote unit + :type cmd: str + :returns: None + :rtype: None + """ + if "sudo" not in cmd: + cmd = "sudo {}".format(cmd) + cmd = ['juju', 'ssh', unit_name, cmd] + logging.info("Running {} on {}".format(cmd, unit_name)) + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + logging.warn("Failed command {} on {}".format(cmd, unit_name)) + logging.warn(e) def do_release_upgrade(unit_name): @@ -376,8 +409,9 @@ def reboot(unit_name): :returns: None :rtype: None """ - # NOTE: Runnig this via model.run_on_unit fails to exit properly - cmd = ['juju', 'run', '--unit', unit_name, 'sudo', 'reboot', '&&', 'exit'] + # NOTE: When used with series upgrade the agent will be down. + # Even juju run will not work + cmd = ['juju', 'ssh', unit_name, 'sudo', 'reboot', '&&', 'exit'] try: subprocess.check_call(cmd) except subprocess.CalledProcessError as e: