arif/zaza-openstack-tests
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create parallel (async) series upgrade

Browse Source
This commit is contained in:
Chris MacNaughton
2020-03-17 17:07:56 +01:00
committed by Chris MacNaughton
parent 93fd631b1a
commit 8450216f72
8 changed files with 754 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@@ -1,5 +1,4 @@
aiounittest
asyncio
async_generator
boto3
juju

32
unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py
View File

@@ -172,38 +172,6 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
{'source': 'new-src'},
model_name=None)
def test__extract_charm_name_from_url(self):
self.assertEqual(
openstack_upgrade._extract_charm_name_from_url(
'local:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade._extract_charm_name_from_url(
'cs:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade._extract_charm_name_from_url('cs:heat'),
'heat')
def test_get_upgrade_candidates(self):
expect = copy.deepcopy(self.juju_status.applications)
del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST
del expect['ntp'] # Filter as it has no source option
del expect['neutron-openvswitch'] # Filter as it is a subordinates
self.assertEqual(
openstack_upgrade.get_upgrade_candidates(),
expect)
def test_get_upgrade_groups(self):
self.assertEqual(
openstack_upgrade.get_upgrade_groups(),
{
'Compute': ['nova-compute'],
'Control Plane': ['cinder'],
'Core Identity': [],
'Storage': [],
'sweep_up': []})
def test_is_action_upgradable(self):
self.assertTrue(
openstack_upgrade.is_action_upgradable('cinder'))

112
unit_tests/utilities/test_zaza_utilities_upgrade_utils.py Normal file
View File

@@ -0,0 +1,112 @@
# Copyright 2020 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import mock
import unit_tests.utils as ut_utils
import zaza.openstack.utilities.upgrade_utils as openstack_upgrade
class TestUpgradeUtils(ut_utils.BaseTestCase):
def setUp(self):
super(TestUpgradeUtils, self).setUp()
self.patch_object(
openstack_upgrade.zaza.model,
"get_units")
self.juju_status = mock.MagicMock()
self.patch_object(
openstack_upgrade.zaza.model,
"get_status",
return_value=self.juju_status)
self.patch_object(
openstack_upgrade.zaza.model,
"get_application_config")
def _get_application_config(app, model_name=None):
app_config = {
'ceph-mon': {'verbose': True, 'source': 'old-src'},
'neutron-openvswitch': {'verbose': True},
'ntp': {'verbose': True},
'percona-cluster': {'verbose': True, 'source': 'old-src'},
'cinder': {
'verbose': True,
'openstack-origin': 'old-src',
'action-managed-upgrade': False},
'neutron-api': {
'verbose': True,
'openstack-origin': 'old-src',
'action-managed-upgrade': False},
'nova-compute': {
'verbose': True,
'openstack-origin': 'old-src',
'action-managed-upgrade': False},
}
return app_config[app]
self.get_application_config.side_effect = _get_application_config
self.juju_status.applications = {
'mydb': { # Filter as it is on UPGRADE_EXCLUDE_LIST
'charm': 'cs:percona-cluster'},
'neutron-openvswitch': { # Filter as it is a subordinates
'charm': 'cs:neutron-openvswitch',
'subordinate-to': 'nova-compute'},
'ntp': { # Filter as it has no source option
'charm': 'cs:ntp'},
'nova-compute': {
'charm': 'cs:nova-compute',
'units': {
'nova-compute/0': {
'subordinates': {
'neutron-openvswitch/2': {
'charm': 'cs:neutron-openvswitch-22'}}}}},
'cinder': {
'charm': 'cs:cinder-23',
'units': {
'cinder/1': {
'subordinates': {
'cinder-hacluster/0': {
'charm': 'cs:hacluster-42'},
'cinder-ceph/3': {
'charm': 'cs:cinder-ceph-2'}}}}}}
def test_get_upgrade_candidates(self):
expect = copy.deepcopy(self.juju_status.applications)
del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST
del expect['ntp'] # Filter as it has no source option
del expect['neutron-openvswitch'] # Filter as it is a subordinates
self.assertEqual(
openstack_upgrade.get_upgrade_candidates(),
expect)
def test_get_upgrade_groups(self):
self.assertEqual(
openstack_upgrade.get_upgrade_groups(),
{
'Core Identity': [],
'Control Plane': ['cinder'],
'Data Plane': ['nova-compute'],
'sweep_up': []})
def test_extract_charm_name_from_url(self):
self.assertEqual(
openstack_upgrade.extract_charm_name_from_url(
'local:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade.extract_charm_name_from_url(
'cs:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade.extract_charm_name_from_url('cs:heat'),
'heat')

190
zaza/openstack/charm_tests/series_upgrade/tests.py
View File

@@ -16,6 +16,7 @@
"""Define class for Series Upgrade."""
import asyncio
import logging
import os
import unittest
@@ -24,6 +25,7 @@ from zaza import model
from zaza.openstack.utilities import (
cli as cli_utils,
series_upgrade as series_upgrade_utils,
upgrade_utils as upgrade_utils,
)
from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest
@@ -54,45 +56,45 @@ class SeriesUpgradeTest(unittest.TestCase):
pause_non_leader_primary = True
post_upgrade_functions = []
# Skip subordinates
if applications[application]["subordinate-to"]:
if app_details["subordinate-to"]:
continue
if "easyrsa" in applications[application]["charm"]:
if "easyrsa" in app_details["charm"]:
logging.warn("Skipping series upgrade of easyrsa Bug #1850121")
continue
if "etcd" in applications[application]["charm"]:
if "etcd" in app_details["charm"]:
logging.warn("Skipping series upgrade of easyrsa Bug #1850124")
continue
if "percona-cluster" in applications[application]["charm"]:
if "percona-cluster" in app_details["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = True
if "rabbitmq-server" in applications[application]["charm"]:
if "rabbitmq-server" in app_details["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = False
if "nova-compute" in applications[application]["charm"]:
if "nova-compute" in app_details["charm"]:
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "ceph" in applications[application]["charm"]:
if "ceph" in app_details["charm"]:
origin = "source"
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "designate-bind" in applications[application]["charm"]:
if "designate-bind" in app_details["charm"]:
origin = None
if "tempest" in applications[application]["charm"]:
if "tempest" in app_details["charm"]:
origin = None
if "memcached" in applications[application]["charm"]:
if "memcached" in app_details["charm"]:
origin = None
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "vault" in applications[application]["charm"]:
if "vault" in app_details["charm"]:
origin = None
pause_non_leader_primary = False
pause_non_leader_subordinate = True
post_upgrade_functions = [
('zaza.openstack.charm_tests.vault.setup.'
'mojo_unseal_by_unit')]
if "mongodb" in applications[application]["charm"]:
if "mongodb" in app_details["charm"]:
# Mongodb needs to run series upgrade
# on its secondaries first.
series_upgrade_utils.series_upgrade_non_leaders_first(
@@ -117,7 +119,7 @@ class SeriesUpgradeTest(unittest.TestCase):
files=self.files,
post_upgrade_functions=post_upgrade_functions)
if "rabbitmq-server" in applications[application]["charm"]:
if "rabbitmq-server" in app_details["charm"]:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
@@ -126,7 +128,7 @@ class SeriesUpgradeTest(unittest.TestCase):
action_params={})
model.block_until_all_units_idle()
if "percona-cluster" in applications[application]["charm"]:
if "percona-cluster" in app_details["charm"]:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
@@ -215,5 +217,165 @@ class XenialBionicSeriesUpgrade(SeriesUpgradeTest):
cls.to_series = "bionic"
class ParallelSeriesUpgradeTest(unittest.TestCase):
"""Class to encapsulate Sereis Upgrade Tests."""
@classmethod
def setUpClass(cls):
"""Run setup for Series Upgrades."""
cli_utils.setup_logging()
cls.from_series = None
cls.to_series = None
cls.workaround_script = None
cls.files = []
def test_200_run_series_upgrade(self):
"""Run series upgrade."""
# Set Feature Flag
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
upgrade_groups = upgrade_utils.get_upgrade_groups()
applications = model.get_status().applications
upgrade_groups['support'] = [
app for app in upgrade_utils.UPGRADE_EXCLUDE_LIST
if app in applications.keys()]
upgrade_groups['deferred'] = []
completed_machines = []
deferred_applications = []
for group_name, group in upgrade_groups.items():
logging.warn("About to upgrade {} ({})".format(group_name, group))
upgrade_group = []
for application, app_details in applications.items():
# Defaults
origin = "openstack-origin"
pause_non_leader_subordinate = True
pause_non_leader_primary = True
post_upgrade_functions = []
name = upgrade_utils.extract_charm_name_from_url(
app_details['charm'])
if name not in group and application not in group:
if group_name is not "deferred" and \
name not in upgrade_groups['deferred']:
upgrade_groups['deferred'].append(name)
continue
if group_name is not "deferred" and \
name in upgrade_groups['deferred']:
upgrade_groups['deferred'].remove(name)
# Skip subordinates
if app_details["subordinate-to"]:
continue
if "easyrsa" in app_details["charm"]:
logging.warn(
"Skipping series upgrade of easyrsa Bug #1850121")
continue
if "etcd" in app_details["charm"]:
logging.warn(
"Skipping series upgrade of easyrsa Bug #1850124")
continue
logging.warn("About to upgrade {}".format(application))
if "percona-cluster" in app_details["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = True
if "rabbitmq-server" in app_details["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = False
if "nova-compute" in app_details["charm"]:
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "ceph" in app_details["charm"]:
origin = "source"
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "designate-bind" in app_details["charm"]:
origin = None
if "tempest" in app_details["charm"]:
origin = None
if "memcached" in app_details["charm"]:
origin = None
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "vault" in app_details["charm"]:
origin = None
pause_non_leader_primary = False
pause_non_leader_subordinate = True
post_upgrade_functions = [
('zaza.openstack.charm_tests.vault.setup.'
'mojo_unseal_by_unit')]
if "mongodb" in app_details["charm"]:
# Mongodb needs to run series upgrade
# on its secondaries first.
upgrade_group.append(series_upgrade_utils.async_series_upgrade_non_leaders_first(
application,
from_series=self.from_series,
to_series=self.to_series,
completed_machines=completed_machines,
post_upgrade_functions=post_upgrade_functions))
continue
# The rest are likley APIs use defaults
upgrade_group.append(series_upgrade_utils.async_series_upgrade_application(
application,
pause_non_leader_primary=pause_non_leader_primary,
pause_non_leader_subordinate=pause_non_leader_subordinate,
from_series=self.from_series,
to_series=self.to_series,
origin=origin,
completed_machines=completed_machines,
workaround_script=self.workaround_script,
files=self.files,
post_upgrade_functions=post_upgrade_functions))
asyncio.get_event_loop().run_until_complete(
asyncio.gather(*upgrade_group, return_exceptions=True))
if "rabbitmq-server" in group:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
'rabbitmq-server',
'complete-cluster-series-upgrade',
action_params={})
model.block_until_all_units_idle()
if "percona-cluster" in group:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
'mysql',
'complete-cluster-series-upgrade',
action_params={})
model.block_until_all_units_idle()
class ParallelTrustyXenialSeriesUpgrade(ParallelSeriesUpgradeTest):
"""Trusty to Xenial Series Upgrade.
Makes no assumptions about what is in the deployment.
"""
@classmethod
def setUpClass(cls):
"""Run setup for Trusty to Xenial Series Upgrades."""
super(ParallelTrustyXenialSeriesUpgrade, cls).setUpClass()
cls.from_series = "trusty"
cls.to_series = "xenial"
class ParallelXenialBionicSeriesUpgrade(ParallelSeriesUpgradeTest):
"""Xenial to Bionic Series Upgrade.
Makes no assumptions about what is in the deployment.
"""
@classmethod
def setUpClass(cls):
"""Run setup for Xenial to Bionic Series Upgrades."""
super(ParallelXenialBionicSeriesUpgrade, cls).setUpClass()
cls.from_series = "xenial"
cls.to_series = "bionic"
if __name__ == "__main__":
unittest.main()

81
zaza/openstack/utilities/generic.py
View File

@@ -14,6 +14,7 @@
"""Collection of functions that did not fit anywhere else."""
import asyncio
import logging
import os
import socket
@@ -205,6 +206,25 @@ def set_origin(application, origin='openstack-origin', pocket='distro'):
model.set_application_config(application, {origin: pocket})
async def async_set_origin(application, origin='openstack-origin',
pocket='distro'):
"""Set the configuration option for origin source.
:param application: Name of application to upgrade series
:type application: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param pocket: Origin source cloud pocket.
i.e. 'distro' or 'cloud:xenial-newton'
:type pocket: str
:returns: None
:rtype: None
"""
logging.info("Set origin on {} to {}".format(application, origin))
await model.async_set_application_config(application, {origin: pocket})
def run_via_ssh(unit_name, cmd):
"""Run command on unit via ssh.
@@ -227,6 +247,28 @@ def run_via_ssh(unit_name, cmd):
logging.warn(e)
async def async_run_via_ssh(unit_name, cmd):
"""Run command on unit via ssh.
For executing commands on units when the juju agent is down.
:param unit_name: Unit Name
:param cmd: Command to execute on remote unit
:type cmd: str
:returns: None
:rtype: None
"""
if "sudo" not in cmd:
# cmd.insert(0, "sudo")
cmd = "sudo {}".format(cmd)
cmd = ['juju', 'ssh', unit_name, cmd]
try:
await check_call(cmd)
except subprocess.CalledProcessError as e:
logging.warn("Failed command {} on {}".format(cmd, unit_name))
logging.warn(e)
def check_commands_on_units(commands, units):
"""Check that all commands in a list exit zero on all units in a list.
@@ -270,6 +312,29 @@ def reboot(unit_name):
pass
async def async_reboot(unit_name):
"""Reboot unit.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
# NOTE: When used with series upgrade the agent will be down.
# Even juju run will not work
await async_run_via_ssh(unit_name, "sudo reboot && exit")
async def check_call(cmd):
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
raise subprocess.CalledProcessError(proc.returncode, cmd)
def set_dpkg_non_interactive_on_unit(
unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"):
"""Set dpkg options on unit.
@@ -286,6 +351,22 @@ def set_dpkg_non_interactive_on_unit(
model.run_on_unit(unit_name, cmd)
async def async_set_dpkg_non_interactive_on_unit(
unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"):
"""Set dpkg options on unit.
:param unit_name: Unit Name
:type unit_name: str
:param apt_conf_d: Apt.conf file to update
:type apt_conf_d: str
"""
DPKG_NON_INTERACTIVE = 'DPkg::options { "--force-confdef"; };'
# Check if the option exists. If not, add it to the apt.conf.d file
cmd = ("grep '{option}' {file_name} || echo '{option}' >> {file_name}"
.format(option=DPKG_NON_INTERACTIVE, file_name=apt_conf_d))
await model.async_run_on_unit(unit_name, cmd)
def get_process_id_list(unit_name, process_name,
expect_success=True):
"""Get a list of process ID(s).

85
zaza/openstack/utilities/openstack_upgrade.py
View File

@@ -16,7 +16,6 @@
This module contains a number of functions for upgrading OpenStack.
"""
import re
import logging
import zaza.openstack.utilities.juju as juju_utils
@@ -25,6 +24,8 @@ from zaza import sync_wrapper
from zaza.openstack.utilities.upgrade_utils import (
SERVICE_GROUPS,
UPGRADE_EXCLUDE_LIST,
get_upgrade_candidates,
get_upgrade_groups,
)
@@ -179,88 +180,6 @@ def set_upgrade_application_config(applications, new_source,
model_name=model_name)
def _extract_charm_name_from_url(charm_url):
"""Extract the charm name from the charm url.
E.g. Extract 'heat' from local:bionic/heat-12
:param charm_url: Name of model to query.
:type charm_url: str
:returns: Charm name
:rtype: str
"""
charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1])
return charm_name.split(':')[-1]
def get_upgrade_candidates(model_name=None):
"""Extract list of apps from model that can be upgraded.
:param model_name: Name of model to query.
:type model_name: str
:returns: List of application that can have their payload upgraded.
:rtype: []
"""
status = zaza.model.get_status(model_name=model_name)
candidates = {}
for app, app_config in status.applications.items():
# Filter out subordinates
if app_config.get("subordinate-to"):
logging.warning(
"Excluding {} from upgrade, it is a subordinate".format(app))
continue
# Filter out charms on the naughty list
charm_name = _extract_charm_name_from_url(app_config['charm'])
if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST:
logging.warning(
"Excluding {} from upgrade, on the exclude list".format(app))
continue
# Filter out charms that have no source option
charm_options = zaza.model.get_application_config(
app, model_name=model_name).keys()
src_options = ['openstack-origin', 'source']
if not [x for x in src_options if x in charm_options]:
logging.warning(
"Excluding {} from upgrade, no src option".format(app))
continue
candidates[app] = app_config
return candidates
def get_upgrade_groups(model_name=None):
"""Place apps in the model into their upgrade groups.
Place apps in the model into their upgrade groups. If an app is deployed
but is not in SERVICE_GROUPS then it is placed in a sweep_up group.
:param model_name: Name of model to query.
:type model_name: str
:returns: Dict of group lists keyed on group name.
:rtype: {}
"""
apps_in_model = get_upgrade_candidates(model_name=model_name)
groups = {}
for phase_name, charms in SERVICE_GROUPS.items():
group = []
for app, app_config in apps_in_model.items():
charm_name = _extract_charm_name_from_url(app_config['charm'])
if charm_name in charms:
group.append(app)
groups[phase_name] = group
sweep_up = []
for app in apps_in_model:
if not (app in [a for group in groups.values() for a in group]):
sweep_up.append(app)
groups['sweep_up'] = sweep_up
return groups
def is_action_upgradable(app, model_name=None):
"""Can application be upgraded using action managed upgrade method.

297
zaza/openstack/utilities/series_upgrade.py
View File

@@ -101,6 +101,68 @@ def series_upgrade_non_leaders_first(application, from_series="trusty",
model.block_until_all_units_idle()
async def async_series_upgrade_non_leaders_first(application, from_series="trusty",
to_series="xenial",
completed_machines=[],
post_upgrade_functions=None):
"""Series upgrade non leaders first.
Wrap all the functionality to handle series upgrade for charms
which must have non leaders upgraded first.
:param application: Name of application to upgrade series
:type application: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:returns: None
:rtype: None
"""
status = (await model.async_get_status()).applications[application]
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Series upgrade the non-leaders first
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
await async_series_upgrade(unit, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
await async_run_post_upgrade_functions(post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
.format(unit, machine, application))
await model.async_block_until_all_units_idle()
# Series upgrade the leader
machine = status["units"][leader]["machine"]
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
await async_series_upgrade(leader, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
.format(unit, machine, application))
await model.async_block_until_all_units_idle()
def series_upgrade_application(application, pause_non_leader_primary=True,
pause_non_leader_subordinate=True,
from_series="trusty", to_series="xenial",
@@ -209,6 +271,144 @@ def series_upgrade_application(application, pause_non_leader_primary=True,
model.block_until_all_units_idle()
async def async_series_upgrade_application(application,
pause_non_leader_primary=True,
pause_non_leader_subordinate=True,
from_series="trusty",
to_series="xenial",
origin='openstack-origin',
completed_machines=None,
files=None, workaround_script=None,
post_upgrade_functions=None):
"""Series upgrade application.
Wrap all the functionality to handle series upgrade for a given
application. Including pausing non-leader units.
:param application: Name of application to upgrade series
:type application: str
:param pause_non_leader_primary: Whether the non-leader applications should
be paused
:type pause_non_leader_primary: bool
:param pause_non_leader_subordinate: Whether the non-leader subordinate
hacluster applications should be
paused
:type pause_non_leader_subordinate: bool
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
if completed_machines is None:
completed_machines = []
status = (await model.async_get_status()).applications[application]
# For some applications (percona-cluster) the leader unit must upgrade
# first. For API applications the non-leader haclusters must be paused
# before upgrade. Finally, for some applications this is arbitrary but
# generalized.
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Pause the non-leaders
for unit in non_leaders:
if pause_non_leader_subordinate:
if status["units"][unit].get("subordinates"):
for subordinate in status["units"][unit]["subordinates"]:
_app = subordinate.split('/')[0]
if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST:
logging.info("Skipping pausing {} - blacklisted"
.format(subordinate))
else:
logging.info("Pausing {}".format(subordinate))
await model.async_run_action(
subordinate, "pause", action_params={})
if pause_non_leader_primary:
logging.info("Pausing {}".format(unit))
await model.async_run_action(unit, "pause", action_params={})
machine = status["units"][leader]["machine"]
# Series upgrade the leader
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
await async_series_upgrade(leader, machine,
from_series=from_series,
to_series=to_series,
origin=origin,
workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
await os_utils.async_set_origin(application, origin)
await wait_for_unit_idle(unit)
# Series upgrade the non-leaders
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
await async_series_upgrade(unit, machine,
from_series=from_series,
to_series=to_series,
origin=origin,
workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
await os_utils.async_set_origin(application, origin)
await wait_for_unit_idle(unit)
async def wait_for_unit_idle(unit_name):
app = unit_name.split('/')[0]
try:
await model.async_block_until(_unit_idle(app, unit_name),
timeout=600)
except concurrent.futures._base.TimeoutError:
raise ModelTimeout("Zaza has timed out waiting on the unit to "
"reach idle state.")
def _unit_idle(app, unit_name):
async def f():
x = await get_agent_status(app, unit_name)
return x == "idle"
return f
#return await get_agent_status(app, unit_name) is "idle"
async def get_agent_status(app, unit_name):
return (await model.async_get_status()). \
applications[app]['units'][unit_name] \
['agent-status']['status']
def series_upgrade(unit_name, machine_num,
from_series="trusty", to_series="xenial",
origin='openstack-origin',
@@ -305,42 +505,93 @@ async def async_series_upgrade(unit_name, machine_num,
application = unit_name.split('/')[0]
await os_utils.async_set_dpkg_non_interactive_on_unit(unit_name)
await async_dist_upgrade(unit_name)
await model.async_block_until_all_units_idle()
await wait_for_unit_idle(unit_name)
logging.info("Prepare series upgrade on {}".format(machine_num))
await model.async_prepare_series_upgrade(machine_num, to_series=to_series)
await async_prepare_series_upgrade(machine_num, to_series=to_series)
logging.info("Waiting for workload status 'blocked' on {}"
.format(unit_name))
await model.async_block_until_unit_wl_status(unit_name, "blocked")
logging.info("Waiting for model idleness")
await model.async_block_until_all_units_idle()
logging.info("Waiting for unit {} idleness".format(unit_name))
await wait_for_unit_idle(unit_name)
await async_wrap_do_release_upgrade(unit_name, from_series=from_series,
to_series=to_series, files=files,
workaround_script=workaround_script)
logging.info("Reboot {}".format(unit_name))
os_utils.reboot(unit_name)
await os_utils.async_reboot(unit_name)
logging.info("Waiting for workload status 'blocked' on {}"
.format(unit_name))
await model.async_block_until_unit_wl_status(unit_name, "blocked")
logging.info("Waiting for model idleness")
await model.async_block_until_all_units_idle()
logging.info("Waiting for unit {} idleness".format(unit_name))
await wait_for_unit_idle(unit_name)
logging.info("Set origin on {}".format(application))
# Allow for charms which have neither source nor openstack-origin
if origin:
await os_utils.async_set_origin(application, origin)
await model.async_block_until_all_units_idle()
await wait_for_unit_idle(unit_name)
logging.info("Complete series upgrade on {}".format(machine_num))
await model.async_complete_series_upgrade(machine_num)
await model.async_block_until_all_units_idle()
await async_complete_series_upgrade(machine_num)
await wait_for_unit_idle(unit_name)
logging.info("Running run_post_upgrade_functions {}".format(
post_upgrade_functions))
run_post_upgrade_functions(post_upgrade_functions)
logging.info("Waiting for workload status 'active' on {}"
.format(unit_name))
await model.async_block_until_unit_wl_status(unit_name, "active")
await model.async_block_until_all_units_idle()
await wait_for_unit_idle(unit_name)
# This step may be performed by juju in the future
logging.info("Set series on {} to {}".format(application, to_series))
await model.async_set_series(application, to_series)
await async_set_series(application, to_series)
async def async_prepare_series_upgrade(machine_num, to_series="xenial"):
"""Execute juju series-upgrade prepare on machine.
NOTE: This is a new feature in juju behind a feature flag and not yet in
libjuju.
export JUJU_DEV_FEATURE_FLAGS=upgrade-series
:param machine_num: Machine number
:type machine_num: str
:param to_series: The series to which to upgrade
:type to_series: str
:returns: None
:rtype: None
"""
juju_model = await model.async_get_juju_model()
cmd = ["juju", "upgrade-series", "-m", juju_model,
machine_num, "prepare", to_series, "--yes"]
logging.info("About to call '{}'".format(cmd))
await os_utils.check_call(cmd)
async def async_complete_series_upgrade(machine_num):
"""Execute juju series-upgrade complete on machine.
NOTE: This is a new feature in juju behind a feature flag and not yet in
libjuju.
export JUJU_DEV_FEATURE_FLAGS=upgrade-series
:param machine_num: Machine number
:type machine_num: str
:returns: None
:rtype: None
"""
juju_model = await model.async_get_juju_model()
cmd = ["juju", "upgrade-series", "-m", juju_model,
machine_num, "complete"]
await os_utils.check_call(cmd)
async def async_set_series(application, to_series):
"""Execute juju set-series complete on application.
NOTE: This is a new feature in juju and not yet in libjuju.
:param application: Name of application to upgrade series
:type application: str
:param to_series: The series to which to upgrade
:type to_series: str
:returns: None
:rtype: None
"""
juju_model = await model.async_get_juju_model()
cmd = ["juju", "set-series", "-m", juju_model,
application, to_series]
await os_utils.check_call(cmd)
def wrap_do_release_upgrade(unit_name, from_series="trusty",
to_series="xenial",
@@ -416,10 +667,10 @@ async def async_wrap_do_release_upgrade(unit_name, from_series="trusty",
# Run Script
if workaround_script:
logging.info("Running workaround script")
os_utils.run_via_ssh(unit_name, workaround_script)
await os_utils.async_run_via_ssh(unit_name, workaround_script)
# Actually do the do_release_upgrade
do_release_upgrade(unit_name)
await async_do_release_upgrade(unit_name)
def dist_upgrade(unit_name):
@@ -477,3 +728,21 @@ def do_release_upgrade(unit_name):
unit_name,
'DEBIAN_FRONTEND=noninteractive '
'do-release-upgrade -f DistUpgradeViewNonInteractive')
async def async_do_release_upgrade(unit_name):
"""Run do-release-upgrade noninteractive.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
logging.info('Upgrading ' + unit_name)
# NOTE: It is necessary to run this via juju ssh rather than juju run due
# to timeout restrictions and error handling.
await os_utils.async_run_via_ssh(
unit_name,
'DEBIAN_FRONTEND=noninteractive '
'do-release-upgrade -f DistUpgradeViewNonInteractive')

109
zaza/openstack/utilities/upgrade_utils.py
View File

@@ -13,18 +13,109 @@
# limitations under the License.
"""Collection of functions to support upgrade testing."""
import re
import logging
import collections
import zaza.model
SERVICE_GROUPS = {
'Core Identity': ['keystone'],
'Storage': [
'ceph-mon', 'ceph-osd', 'ceph-fs', 'ceph-radosgw', 'swift-proxy',
'swift-storage'],
'Control Plane': [
'aodh', 'barbican', 'ceilometer', 'cinder', 'designate',
SERVICE_GROUPS = collections.OrderedDict([
('Core Identity', ['keystone']),
('Control Plane', [
'aodh', 'barbican', 'ceilometer', 'ceph-mon', 'ceph-fs',
'ceph-radosgw', 'cinder', 'designate',
'designate-bind', 'glance', 'gnocchi', 'heat', 'manila',
'manila-generic', 'neutron-api', 'neutron-gateway', 'placement',
'nova-cloud-controller', 'openstack-dashboard'],
'Compute': ['nova-compute']}
'nova-cloud-controller', 'openstack-dashboard']),
('Data Plane', [
'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage'])
])
UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster']
# Series upgrade ordering should be: [
# UPGRADE_EXCLUDE_LIST,
# SERVICE_GROUPS['Core Identity'],
# SERVICE_GROUPS['Control Plane'] + SERVICE_GROUPS['Storage'],
# SERVICE_GROUPS['Data Plane'],
# ]
def get_upgrade_candidates(model_name=None):
"""Extract list of apps from model that can be upgraded.
:param model_name: Name of model to query.
:type model_name: str
:returns: List of application that can have their payload upgraded.
:rtype: []
"""
status = zaza.model.get_status(model_name=model_name)
candidates = {}
for app, app_config in status.applications.items():
# Filter out subordinates
if app_config.get("subordinate-to"):
logging.warning(
"Excluding {} from upgrade, it is a subordinate".format(app))
continue
# Filter out charms on the naughty list
charm_name = extract_charm_name_from_url(app_config['charm'])
if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST:
logging.warning(
"Excluding {} from upgrade, on the exclude list".format(app))
continue
# Filter out charms that have no source option
charm_options = zaza.model.get_application_config(
app, model_name=model_name).keys()
src_options = ['openstack-origin', 'source']
if not [x for x in src_options if x in charm_options]:
logging.warning(
"Excluding {} from upgrade, no src option".format(app))
continue
candidates[app] = app_config
return candidates
def get_upgrade_groups(model_name=None):
"""Place apps in the model into their upgrade groups.
Place apps in the model into their upgrade groups. If an app is deployed
but is not in SERVICE_GROUPS then it is placed in a sweep_up group.
:param model_name: Name of model to query.
:type model_name: str
:returns: Dict of group lists keyed on group name.
:rtype: collections.OrderedDict
"""
apps_in_model = get_upgrade_candidates(model_name=model_name)
groups = collections.OrderedDict()
for phase_name, charms in SERVICE_GROUPS.items():
group = []
for app, app_config in apps_in_model.items():
charm_name = extract_charm_name_from_url(app_config['charm'])
if charm_name in charms:
group.append(app)
groups[phase_name] = group
sweep_up = []
for app in apps_in_model:
if not (app in [a for group in groups.values() for a in group]):
sweep_up.append(app)
groups['sweep_up'] = sweep_up
return groups
def extract_charm_name_from_url(charm_url):
"""Extract the charm name from the charm url.
E.g. Extract 'heat' from local:bionic/heat-12
:param charm_url: Name of model to query.
:type charm_url: str
:returns: Charm name
:rtype: str
"""
charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1])
return charm_name.split(':')[-1]