From 2a98274d67589a5a251c0fa9082f28e27abc6833 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 9 Apr 2020 10:56:31 +0200 Subject: [PATCH] Add a few more function hooks to the series upgrade process By including pre and pos-application functions, charm series upgrades can be handled in a more generic way, even when they require running additional actions before a unit is upgraded, or after the whole application is upgraded. --- zaza/openstack/charm_tests/mysql/utils.py | 26 +++++ .../charm_tests/rabbitmq_server/utils.py | 8 ++ .../series_upgrade/parallel_tests.py | 16 ---- .../utilities/parallel_series_upgrade.py | 94 +++++++++++++++++-- zaza/openstack/utilities/upgrade_utils.py | 4 +- 5 files changed, 123 insertions(+), 25 deletions(-) create mode 100644 zaza/openstack/charm_tests/mysql/utils.py diff --git a/zaza/openstack/charm_tests/mysql/utils.py b/zaza/openstack/charm_tests/mysql/utils.py new file mode 100644 index 0000000..2622903 --- /dev/null +++ b/zaza/openstack/charm_tests/mysql/utils.py @@ -0,0 +1,26 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module of functions for interfacing with the percona-cluster charm.""" + +import zaza.model as model + + +def complete_cluster_series_upgrade(): + """Run the complete-cluster-series-upgrade action on the lead unit.""" + # TODO: Make this work across either mysql or percona-cluster names + model.run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 9826add..62db7e6 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -584,3 +584,11 @@ def _post_check_unit_cluster_nodes(unit, nodes, unit_node_names): 'after unit removal.\n' ''.format(unit_name, node)) return errors + + +def complete_cluster_series_upgrade(): + """Run the complete-cluster-series-upgrade action on the lead unit.""" + zaza.model.run_action_on_leader( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index 7c9dba0..5307339 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -101,22 +101,6 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): files=files)) asyncio.get_event_loop().run_until_complete( asyncio.gather(*upgrade_functions)) - - if "rabbitmq-server" in apps: - logging.info( - "Running complete-cluster-series-upgrade action on leader") - model.run_action_on_leader( - 'rabbitmq-server', - 'complete-cluster-series-upgrade', - action_params={}) - - if "percona-cluster" in apps: - logging.info( - "Running complete-cluster-series-upgrade action on leader") - model.run_action_on_leader( - 'mysql', - 'complete-cluster-series-upgrade', - action_params={}) model.block_until_all_units_idle() logging.info("Finished {}".format(group_name)) logging.info("Done!") diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 1585ef7..622c5ac 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -23,6 +23,7 @@ import logging import subprocess from zaza import model +from zaza.charm_lifecycle import utils as cl_utils import zaza.openstack.utilities.generic as os_utils import zaza.openstack.utilities.series_upgrade as series_upgrade_utils from zaza.openstack.utilities.series_upgrade import ( @@ -45,6 +46,8 @@ def app_config(charm_name): 'pause_non_leader_subordinate': True, 'pause_non_leader_primary': True, 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], 'follower_first': False, } _app_settings = collections.defaultdict(lambda: default) ceph = { @@ -55,11 +58,25 @@ def app_config(charm_name): exceptions = { 'rabbitmq-server': { 'origin': 'source', - 'pause_non_leader_subordinate': False, }, - 'percona-cluster': {'origin': 'source', }, + 'pause_non_leader_subordinate': False, + 'post_application_upgrade_functions': [ + ('zaza.openstack.charm_tests.rabbitmq_server.utils.' + 'complete_cluster_series_upgrade')] + }, + 'percona-cluster': { + 'origin': 'source', + 'post_application_upgrade_functions': [ + ('zaza.openstack.charm_tests.mysql.utils.' + 'complete_cluster_series_upgrade')] + }, 'nova-compute': { 'pause_non_leader_primary': False, - 'pause_non_leader_subordinate': False, }, + 'pause_non_leader_subordinate': False, + # TODO + # 'pre_upgrade_functions': [ + # 'zaza.openstack.charm_tests.nova_compute.setup.evacuate' + # ] + }, 'ceph': ceph, 'ceph-mon': ceph, 'ceph-osd': ceph, @@ -111,7 +128,8 @@ async def parallel_series_upgrade( origin='openstack-origin', pause_non_leader_primary=True, pause_non_leader_subordinate=True, post_upgrade_functions=None, completed_machines=None, files=None, workaround_script=None, - follower_first=False): + follower_first=False, pre_upgrade_functions=None, + post_application_upgrade_functions=None): """Perform series upgrade on an application in parallel. :param unit_name: Unit Name @@ -132,9 +150,17 @@ async def parallel_series_upgrade( hacluster applications should be paused :type pause_non_leader_subordinate: bool + + :param pre_upgrade_functions: A list of Zaza functions to call before + the upgrade is started on each machine + :type pre_upgrade_functions: List[str] :param post_upgrade_functions: A list of Zaza functions to call when the upgrade is complete on each machine :type post_upgrade_functions: List[str] + :param post_application_upgrade_functions: A list of Zaza functions + to call when the upgrade is complete + on all machine in the application + :type post_application_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -150,12 +176,16 @@ async def parallel_series_upgrade( files = [] if post_upgrade_functions is None: post_upgrade_functions = [] + if pre_upgrade_functions is None: + pre_upgrade_functions = [] + if post_application_upgrade_functions is None: + post_application_upgrade_functions = [] if follower_first: logging.error("leader_first is ignored for parallel upgrade") logging.info( "About to upgrade the units of {} in parallel (follower first: {})" .format(application, follower_first)) - # return + status = (await model.async_get_status()).applications[application] leader, non_leaders = await get_leader_and_non_leaders(application) for leader_name, leader_unit in leader.items(): @@ -165,6 +195,7 @@ async def parallel_series_upgrade( unit["machine"] for name, unit in non_leaders.items() if unit['machine'] not in completed_machines] + await maybe_pause_things( status, non_leaders, @@ -190,6 +221,7 @@ async def parallel_series_upgrade( await asyncio.gather(*upgrade_group) if origin: await os_utils.async_set_origin(application, origin) + run_post_application_upgrade_functions(post_application_upgrade_functions) async def serial_series_upgrade( @@ -197,7 +229,8 @@ async def serial_series_upgrade( origin='openstack-origin', pause_non_leader_primary=True, pause_non_leader_subordinate=True, post_upgrade_functions=None, completed_machines=None, files=None, workaround_script=None, - follower_first=False,): + follower_first=False, pre_upgrade_functions=None, + post_application_upgrade_functions=None): """Perform series upgrade on an application in series. :param unit_name: Unit Name @@ -218,9 +251,16 @@ async def serial_series_upgrade( hacluster applications should be paused :type pause_non_leader_subordinate: bool + :param pre_upgrade_functions: A list of Zaza functions to call before + the upgrade is started on each machine + :type pre_upgrade_functions: List[str] :param post_upgrade_functions: A list of Zaza functions to call when the upgrade is complete on each machine :type post_upgrade_functions: List[str] + :param post_application_upgrade_functions: A list of Zaza functions + to call when the upgrade is complete + on all machine in the application + :type post_application_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -236,6 +276,10 @@ async def serial_series_upgrade( files = [] if post_upgrade_functions is None: post_upgrade_functions = [] + if pre_upgrade_functions is None: + pre_upgrade_functions = [] + if post_application_upgrade_functions is None: + post_application_upgrade_functions = [] logging.info( "About to upgrade the units of {} in serial (follower first: {})" .format(application, follower_first)) @@ -287,11 +331,13 @@ async def serial_series_upgrade( post_upgrade_functions=post_upgrade_functions) if origin: await os_utils.async_set_origin(application, origin) + run_post_application_upgrade_functions(post_application_upgrade_functions) async def series_upgrade_machine( machine, from_series='xenial', to_series='bionic', - files=None, workaround_script=None, post_upgrade_functions=None): + files=None, workaround_script=None, post_upgrade_functions=None, + pre_upgrade_functions=None): """Perform series upgrade on an machine. :param machine_num: Machine number @@ -304,6 +350,9 @@ async def series_upgrade_machine( :type files: list :param workaround_script: Workaround script to run during series upgrade :type workaround_script: str + :param pre_upgrade_functions: A list of Zaza functions to call before + the upgrade is started on each machine + :type pre_upgrade_functions: List[str] :param post_upgrade_functions: A list of Zaza functions to call when the upgrade is complete on each machine :type post_upgrade_functions: List[str] @@ -312,6 +361,8 @@ async def series_upgrade_machine( """ logging.info( "About to dist-upgrade ({})".format(machine)) + + run_pre_upgrade_functions(pre_upgrade_functions) # upgrade the do the dist upgrade await async_dist_upgrade(machine) # do a do-release-upgrade @@ -322,6 +373,35 @@ async def series_upgrade_machine( series_upgrade_utils.run_post_upgrade_functions(post_upgrade_functions) +def run_pre_upgrade_functions(machine, pre_upgrade_functions): + """Execute list supplied functions. + + Each of the supplied functions will be called with a single + argument of the machine that is about to be upgraded. + + :param machine: Machine that is about to be upgraded + :type machine: str + :param pre_upgrade_functions: List of functions + :type pre_upgrade_functions: [function, function, ...] + """ + if pre_upgrade_functions: + for func in pre_upgrade_functions: + logging.info("Running {}".format(func)) + cl_utils.get_class(func)(machine) + + +def run_post_application_upgrade_functions(post_upgrade_functions): + """Execute list supplied functions. + + :param post_upgrade_functions: List of functions + :type post_upgrade_functions: [function, function, ...] + """ + if post_upgrade_functions: + for func in post_upgrade_functions: + logging.info("Running {}".format(func)) + cl_utils.get_class(func)() + + async def maybe_pause_things( status, units, pause_non_leader_subordinate=True, pause_non_leader_primary=True): diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index b24aec7..c864b9b 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -21,7 +21,7 @@ import zaza.model SERVICE_GROUPS = collections.OrderedDict([ ('Stateful Services', ['percona-cluster', 'rabbitmq-server']), - ('Core Identity', ['keystone']), + ('Core Identity', ['keystone', 'ceph-mon']), ('Control Plane', [ 'aodh', 'barbican', 'ceilometer', 'ceph-fs', 'ceph-radosgw', 'cinder', 'designate', @@ -29,7 +29,7 @@ SERVICE_GROUPS = collections.OrderedDict([ 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', 'nova-cloud-controller', 'openstack-dashboard']), ('Data Plane', [ - 'nova-compute', 'ceph-mon', 'ceph-osd', + 'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage']) ])