From 66d95d65d7637b6b4f56c575ed552f13ebfa02b3 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Fri, 4 Sep 2020 12:33:42 +0100 Subject: [PATCH] Fix series-upgrade tests (#406) A couple of changes here: * Ensure that the post-upgrade-hook runs BEFORE the config-changed to set the openstack-origin/source back to distro. The former behaviour breaks keystone quite badly. * Ensure that the charm name is used, as discovered from the model, for rabbitmq-server and percona-cluster to cope with different names for the application in the model vs the charm name from the charm-store. * Check whether the machine needs to be rebooted after the dist-upgrade (before the do-release-upgrade), and reboot the machine first. Otherwise, do-release-upgrade will fail. --- requirements.txt | 2 +- .../charm_tests/series_upgrade/tests.py | 4 +- zaza/openstack/utilities/series_upgrade.py | 38 +++++++++++++++++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index fa0e460..8b15edc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiounittest async_generator boto3 -juju +juju!=2.8.3 # blacklist 2.8.3 as it appears to have a connection bug juju_wait PyYAML<=4.2,>=3.0 flake8>=2.2.4 diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 4b06dbb..c595448 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -81,7 +81,7 @@ class SeriesUpgradeTest(unittest.TestCase): logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( - 'rabbitmq-server', + charm_name, 'complete-cluster-series-upgrade', action_params={}) model.block_until_all_units_idle() @@ -90,7 +90,7 @@ class SeriesUpgradeTest(unittest.TestCase): logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( - 'mysql', + charm_name, 'complete-cluster-series-upgrade', action_params={}) model.block_until_all_units_idle() diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index f12f77b..97ba153 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -14,11 +14,13 @@ """Collection of functions for testing series upgrade.""" +import asyncio import collections import copy import concurrent import logging import os +import time from zaza import model from zaza.charm_lifecycle import utils as cl_utils @@ -642,14 +644,14 @@ def series_upgrade(unit_name, machine_num, model.block_until_unit_wl_status(unit_name, "blocked") logging.info("Waiting for model idleness") model.block_until_all_units_idle() + logging.info("Complete series upgrade on {}".format(machine_num)) + model.complete_series_upgrade(machine_num) + model.block_until_all_units_idle() logging.info("Set origin on {}".format(application)) # Allow for charms which have neither source nor openstack-origin if origin: os_utils.set_origin(application, origin) model.block_until_all_units_idle() - logging.info("Complete series upgrade on {}".format(machine_num)) - model.complete_series_upgrade(machine_num) - model.block_until_all_units_idle() logging.info("Running run_post_upgrade_functions {}".format( post_upgrade_functions)) run_post_upgrade_functions(post_upgrade_functions) @@ -882,6 +884,21 @@ def dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") model.run_on_unit(unit_name, dist_upgrade_cmd) + rdict = model.run_on_unit(unit_name, "cat /var/run/reboot-required") + if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): + logging.info("dist-upgrade required reboot {}".format(unit_name)) + os_utils.reboot(unit_name) + logging.info("Waiting for workload status 'unknown' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "unknown") + logging.info("Waiting for workload status to return to normal on {}" + .format(unit_name)) + model.block_until_unit_wl_status( + unit_name, "unknown", negate_match=True) + logging.info("Waiting for model idleness") + # pause for a big + time.sleep(5.0) + model.block_until_all_units_idle() async def async_dist_upgrade(unit_name): @@ -902,6 +919,21 @@ async def async_dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") await model.async_run_on_unit(unit_name, dist_upgrade_cmd) + rdict = await model.async_run_on_unit(unit_name, + "cat /var/run/reboot-required") + if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): + logging.info("dist-upgrade required reboot {}".format(unit_name)) + await os_utils.async_reboot(unit_name) + logging.info("Waiting for workload status 'unknown' on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status(unit_name, "unknown") + logging.info("Waiting for workload status to return to normal on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status( + unit_name, "unknown", negate_match=True) + logging.info("Waiting for model idleness") + await asyncio.sleep(5.0) + await model.async_block_until_all_units_idle() def do_release_upgrade(unit_name):