arif/zaza-openstack-tests
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #145 from gnuoy/add-openstack-upgrade-lib

Browse Source 
Add functions for performing OpenStack upgrades
This commit is contained in:
Alex Kavanagh
2020-02-04 11:42:54 +00:00
committed by GitHub
parent 6f446a2a42 4af37d9556
commit 7538791b52
2 changed files with 644 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

273
unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py Normal file
View File

@@ -0,0 +1,273 @@
# Copyright 2019 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import mock
import unit_tests.utils as ut_utils
import zaza.openstack.utilities.openstack_upgrade as openstack_upgrade
class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
async def _arun_action_on_units(self, units, cmd, model_name=None,
raise_on_failure=True):
pass
def setUp(self):
super(TestOpenStackUpgradeUtils, self).setUp()
self.patch_object(
openstack_upgrade.zaza.model,
"async_run_action_on_units")
self.async_run_action_on_units.side_effect = self._arun_action_on_units
self.patch_object(
openstack_upgrade.zaza.model,
"get_units")
self.juju_status = mock.MagicMock()
self.patch_object(
openstack_upgrade.zaza.model,
"get_status",
return_value=self.juju_status)
self.patch_object(
openstack_upgrade.zaza.model,
"set_application_config")
self.patch_object(
openstack_upgrade.zaza.model,
"get_application_config")
def _get_application_config(app, model_name=None):
app_config = {
'ceph-mon': {'verbose': True, 'source': 'old-src'},
'neutron-openvswitch': {'verbose': True},
'ntp': {'verbose': True},
'percona-cluster': {'verbose': True, 'source': 'old-src'},
'cinder': {
'verbose': True,
'openstack-origin': 'old-src',
'action-managed-upgrade': False},
'neutron-api': {
'verbose': True,
'openstack-origin': 'old-src',
'action-managed-upgrade': False},
'nova-compute': {
'verbose': True,
'openstack-origin': 'old-src',
'action-managed-upgrade': False},
}
return app_config[app]
self.get_application_config.side_effect = _get_application_config
self.juju_status.applications = {
'mydb': { # Filter as it is on UPGRADE_EXCLUDE_LIST
'charm': 'cs:percona-cluster'},
'neutron-openvswitch': { # Filter as it is a subordinates
'charm': 'cs:neutron-openvswitch',
'subordinate-to': 'nova-compute'},
'ntp': { # Filter as it has no source option
'charm': 'cs:ntp'},
'nova-compute': {
'charm': 'cs:nova-compute',
'units': {
'nova-compute/0': {
'subordinates': {
'neutron-openvswitch/2': {
'charm': 'cs:neutron-openvswitch-22'}}}}},
'cinder': {
'charm': 'cs:cinder-23',
'units': {
'cinder/1': {
'subordinates': {
'cinder-hacluster/0': {
'charm': 'cs:hacluster-42'},
'cinder-ceph/3': {
'charm': 'cs:cinder-ceph-2'}}}}}}
def test_pause_units(self):
openstack_upgrade.pause_units(['cinder/1', 'glance/2'])
self.async_run_action_on_units.assert_called_once_with(
['cinder/1', 'glance/2'],
'pause',
model_name=None,
raise_on_failure=True)
def test_resume_units(self):
openstack_upgrade.resume_units(['cinder/1', 'glance/2'])
self.async_run_action_on_units.assert_called_once_with(
['cinder/1', 'glance/2'],
'resume',
model_name=None,
raise_on_failure=True)
def test_action_unit_upgrade(self):
openstack_upgrade.action_unit_upgrade(['cinder/1', 'glance/2'])
self.async_run_action_on_units.assert_called_once_with(
['cinder/1', 'glance/2'],
'openstack-upgrade',
model_name=None,
raise_on_failure=True)
def test_action_upgrade_group(self):
self.patch_object(openstack_upgrade, "pause_units")
self.patch_object(openstack_upgrade, "action_unit_upgrade")
self.patch_object(openstack_upgrade, "resume_units")
mock_nova_compute_0 = mock.MagicMock()
mock_nova_compute_0.entity_id = 'nova-compute/0'
mock_cinder_1 = mock.MagicMock()
mock_cinder_1.entity_id = 'cinder/1'
units = {
'nova-compute': [mock_nova_compute_0],
'cinder': [mock_cinder_1]}
self.get_units.side_effect = lambda app, model_name: units[app]
openstack_upgrade.action_upgrade_group(['nova-compute', 'cinder'])
pause_calls = [
mock.call(['cinder-hacluster/0'], model_name=None),
mock.call(['nova-compute/0', 'cinder/1'], model_name=None)]
self.pause_units.assert_has_calls(pause_calls, any_order=False)
action_unit_upgrade_calls = [
mock.call(['nova-compute/0', 'cinder/1'], model_name=None)]
self.action_unit_upgrade.assert_has_calls(
action_unit_upgrade_calls,
any_order=False)
resume_calls = [
mock.call(['nova-compute/0', 'cinder/1'], model_name=None),
mock.call(['cinder-hacluster/0'], model_name=None)]
self.resume_units.assert_has_calls(resume_calls, any_order=False)
def test_set_upgrade_application_config(self):
openstack_upgrade.set_upgrade_application_config(
['neutron-api', 'cinder'],
'new-src')
set_app_calls = [
mock.call(
'neutron-api',
{
'openstack-origin': 'new-src',
'action-managed-upgrade': 'True'},
model_name=None),
mock.call(
'cinder',
{
'openstack-origin': 'new-src',
'action-managed-upgrade': 'True'},
model_name=None)]
self.set_application_config.assert_has_calls(set_app_calls)
self.set_application_config.reset_mock()
openstack_upgrade.set_upgrade_application_config(
['percona-cluster'],
'new-src',
action_managed=False)
self.set_application_config.assert_called_once_with(
'percona-cluster',
{'source': 'new-src'},
model_name=None)
def test__extract_charm_name_from_url(self):
self.assertEqual(
openstack_upgrade._extract_charm_name_from_url(
'local:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade._extract_charm_name_from_url(
'cs:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade._extract_charm_name_from_url('cs:heat'),
'heat')
def test_get_upgrade_candidates(self):
expect = copy.deepcopy(self.juju_status.applications)
del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST
del expect['ntp'] # Filter as it has no source option
del expect['neutron-openvswitch'] # Filter as it is a subordinates
self.assertEqual(
openstack_upgrade.get_upgrade_candidates(),
expect)
def test_get_upgrade_groups(self):
self.assertEqual(
openstack_upgrade.get_upgrade_groups(),
{
'Compute': ['nova-compute'],
'Control Plane': ['cinder'],
'Core Identity': [],
'Storage': [],
'sweep_up': []})
def test_is_action_upgradable(self):
self.assertTrue(
openstack_upgrade.is_action_upgradable('cinder'))
self.assertFalse(
openstack_upgrade.is_action_upgradable('percona-cluster'))
def test_run_action_upgrade(self):
self.patch_object(openstack_upgrade, "set_upgrade_application_config")
self.patch_object(openstack_upgrade, "action_upgrade_group")
openstack_upgrade.run_action_upgrade(
['cinder', 'neutron-api'],
'new-src')
self.set_upgrade_application_config.assert_called_once_with(
['cinder', 'neutron-api'],
'new-src',
model_name=None)
self.action_upgrade_group.assert_called_once_with(
['cinder', 'neutron-api'],
model_name=None)
def test_run_all_in_one_upgrade(self):
self.patch_object(openstack_upgrade, "set_upgrade_application_config")
self.patch_object(
openstack_upgrade.zaza.model,
'block_until_all_units_idle')
openstack_upgrade.run_all_in_one_upgrade(
['percona-cluster'],
'new-src')
self.set_upgrade_application_config.assert_called_once_with(
['percona-cluster'],
'new-src',
action_managed=False,
model_name=None)
self.block_until_all_units_idle.assert_called_once_with()
def test_run_upgrade(self):
self.patch_object(openstack_upgrade, "run_all_in_one_upgrade")
self.patch_object(openstack_upgrade, "run_action_upgrade")
openstack_upgrade.run_upgrade(
['cinder', 'neutron-api', 'ceph-mon'],
'new-src')
self.run_all_in_one_upgrade.assert_called_once_with(
['ceph-mon'],
'new-src',
model_name=None)
self.run_action_upgrade.assert_called_once_with(
['cinder', 'neutron-api'],
'new-src',
model_name=None)
def test_run_upgrade_tests(self):
self.patch_object(openstack_upgrade, "run_upgrade")
self.patch_object(openstack_upgrade, "get_upgrade_groups")
self.get_upgrade_groups.return_value = {
'Compute': ['nova-compute'],
'Control Plane': ['cinder', 'neutron-api'],
'Core Identity': ['keystone'],
'Storage': ['ceph-mon'],
'sweep_up': ['designate']}
openstack_upgrade.run_upgrade_tests('new-src', model_name=None)
run_upgrade_calls = [
mock.call(['keystone'], 'new-src', model_name=None),
mock.call(['ceph-mon'], 'new-src', model_name=None),
mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None),
mock.call(['nova-compute'], 'new-src', model_name=None),
mock.call(['designate'], 'new-src', model_name=None)]
self.run_upgrade.assert_has_calls(run_upgrade_calls, any_order=False)

371
zaza/openstack/utilities/openstack_upgrade.py Executable file
View File

@@ -0,0 +1,371 @@
# Copyright 2019 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for performing OpenStack upgrades.
This module contains a number of functions for upgrading OpenStack.
"""
import re
import logging
import zaza.openstack.utilities.juju as juju_utils
import zaza.model
from zaza import sync_wrapper
SERVICE_GROUPS = {
'Core Identity': ['keystone'],
'Storage': [
'ceph-mon', 'ceph-osd', 'ceph-fs', 'ceph-radosgw', 'swift-proxy',
'swift-storage'],
'Control Plane': [
'aodh', 'barbican', 'ceilometer', 'cinder', 'designate',
'designate-bind', 'glance', 'gnocchi', 'heat', 'manila',
'manila-generic', 'neutron-api', 'neutron-gateway', 'placement',
'nova-cloud-controller', 'openstack-dashboard'],
'Compute': ['nova-compute']}
UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster']
async def async_pause_units(units, model_name=None):
"""Pause all units in unit list.
Pause all units in unit list. Wait for pause action
to complete.
:param units: List of unit names.
:type units: []
:param model_name: Name of model to query.
:type model_name: str
:rtype: juju.action.Action
:raises: zaza.model.ActionFailed
"""
logging.info("Pausing {}".format(', '.join(units)))
await zaza.model.async_run_action_on_units(
units,
'pause',
model_name=model_name,
raise_on_failure=True)
pause_units = sync_wrapper(async_pause_units)
async def async_resume_units(units, model_name=None):
"""Resume all units in unit list.
Resume all units in unit list. Wait for resume action
to complete.
:param units: List of unit names.
:type units: []
:param model_name: Name of model to query.
:type model_name: str
:rtype: juju.action.Action
:raises: zaza.model.ActionFailed
"""
logging.info("Resuming {}".format(', '.join(units)))
await zaza.model.async_run_action_on_units(
units,
'resume',
model_name=model_name,
raise_on_failure=True)
resume_units = sync_wrapper(async_resume_units)
async def async_action_unit_upgrade(units, model_name=None):
"""Run openstack-upgrade on all units in unit list.
Upgrade payload on all units in unit list. Wait for action
to complete.
:param units: List of unit names.
:type units: []
:param model_name: Name of model to query.
:type model_name: str
:rtype: juju.action.Action
:raises: zaza.model.ActionFailed
"""
logging.info("Upgrading {}".format(', '.join(units)))
await zaza.model.async_run_action_on_units(
units,
'openstack-upgrade',
model_name=model_name,
raise_on_failure=True)
action_unit_upgrade = sync_wrapper(async_action_unit_upgrade)
def action_upgrade_group(applications, model_name=None):
"""Upgrade units using action managed upgrades.
Upgrade all units of the given applications using action managed upgrades.
This involves the following process:
1) Take a unit from each application which has not been upgraded yet.
2) Pause all hacluster units assocaiated with units to be upgraded.
3) Pause target units.
4) Upgrade target units.
5) Resume target units.
6) Resume hacluster units paused in step 2.
7) Repeat until all units are upgraded.
:param applications: List of application names.
:type applications: []
:param model_name: Name of model to query.
:type model_name: str
"""
status = zaza.model.get_status(model_name=model_name)
done = []
while True:
target = []
for app in applications:
for unit in zaza.model.get_units(app, model_name=model_name):
if unit.entity_id not in done:
target.append(unit.entity_id)
break
else:
logging.info("All units of {} upgraded".format(app))
if not target:
break
hacluster_units = juju_utils.get_subordinate_units(
target,
'hacluster',
status=status,
model_name=model_name)
pause_units(hacluster_units, model_name=model_name)
pause_units(target, model_name=model_name)
action_unit_upgrade(target, model_name=model_name)
resume_units(target, model_name=model_name)
resume_units(hacluster_units, model_name=model_name)
done.extend(target)
def set_upgrade_application_config(applications, new_source,
action_managed=True, model_name=None):
"""Set the charm config for upgrade.
Set the charm config for upgrade.
:param applications: List of application names.
:type applications: []
:param new_source: New package origin.
:type new_source: str
:param action_managed: Whether to set action-managed-upgrade config option.
:type action_managed: bool
:param model_name: Name of model to query.
:type model_name: str
"""
for app in applications:
src_option = 'openstack-origin'
charm_options = zaza.model.get_application_config(
app, model_name=model_name)
try:
charm_options[src_option]
except KeyError:
src_option = 'source'
config = {
src_option: new_source}
if action_managed:
config['action-managed-upgrade'] = 'True'
logging.info("Setting config for {} to {}".format(app, config))
zaza.model.set_application_config(
app,
config,
model_name=model_name)
def _extract_charm_name_from_url(charm_url):
"""Extract the charm name from the charm url.
E.g. Extract 'heat' from local:bionic/heat-12
:param charm_url: Name of model to query.
:type charm_url: str
:returns: Charm name
:rtype: str
"""
charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1])
return charm_name.split(':')[-1]
def get_upgrade_candidates(model_name=None):
"""Extract list of apps from model that can be upgraded.
:param model_name: Name of model to query.
:type model_name: str
:returns: List of application that can have their payload upgraded.
:rtype: []
"""
status = zaza.model.get_status(model_name=model_name)
candidates = {}
for app, app_config in status.applications.items():
# Filter out subordinates
if app_config.get("subordinate-to"):
logging.warning(
"Excluding {} from upgrade, it is a subordinate".format(app))
continue
# Filter out charms on the naughty list
charm_name = _extract_charm_name_from_url(app_config['charm'])
if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST:
logging.warning(
"Excluding {} from upgrade, on the exclude list".format(app))
continue
# Filter out charms that have no source option
charm_options = zaza.model.get_application_config(
app, model_name=model_name).keys()
src_options = ['openstack-origin', 'source']
if not [x for x in src_options if x in charm_options]:
logging.warning(
"Excluding {} from upgrade, no src option".format(app))
continue
candidates[app] = app_config
return candidates
def get_upgrade_groups(model_name=None):
"""Place apps in the model into their upgrade groups.
Place apps in the model into their upgrade groups. If an app is deployed
but is not in SERVICE_GROUPS then it is placed in a sweep_up group.
:param model_name: Name of model to query.
:type model_name: str
:returns: Dict of group lists keyed on group name.
:rtype: {}
"""
apps_in_model = get_upgrade_candidates(model_name=model_name)
groups = {}
for phase_name, charms in SERVICE_GROUPS.items():
group = []
for app, app_config in apps_in_model.items():
charm_name = _extract_charm_name_from_url(app_config['charm'])
if charm_name in charms:
group.append(app)
groups[phase_name] = group
sweep_up = []
for app in apps_in_model:
if not (app in [a for group in groups.values() for a in group]):
sweep_up.append(app)
groups['sweep_up'] = sweep_up
return groups
def is_action_upgradable(app, model_name=None):
"""Can application be upgraded using action managed upgrade method.
:param new_source: New package origin.
:type new_source: str
:param model_name: Name of model to query.
:type model_name: str
:returns: Whether app be upgraded using action managed upgrade method.
:rtype: bool
"""
config = zaza.model.get_application_config(app, model_name=model_name)
try:
config['action-managed-upgrade']
supported = True
except KeyError:
supported = False
return supported
def run_action_upgrade(group, new_source, model_name=None):
"""Upgrade payload of all applications in group using action upgrades.
:param group: List of applications to upgrade.
:type group
:param new_source: New package origin.
:type new_source: str
:param model_name: Name of model to query.
:type model_name: str
"""
set_upgrade_application_config(group, new_source, model_name=model_name)
action_upgrade_group(group, model_name=model_name)
def run_all_in_one_upgrade(group, new_source, model_name=None):
"""Upgrade payload of all applications in group using all-in-one method.
:param group: List of applications to upgrade.
:type group: []
:source: New package origin.
:type new_source: str
:param model_name: Name of model to query.
:type model_name: str
"""
set_upgrade_application_config(
group,
new_source,
model_name=model_name,
action_managed=False)
zaza.model.block_until_all_units_idle()
def run_upgrade(group, new_source, model_name=None):
"""Upgrade payload of all applications in group.
Upgrade apps using action managed upgrades where possible and fallback to
all_in_one method.
:param group: List of applications to upgrade.
:type group: []
:param new_source: New package origin.
:type new_source: str
:param model_name: Name of model to query.
:type model_name: str
"""
action_upgrade = []
all_in_one_upgrade = []
for app in group:
if is_action_upgradable(app, model_name=model_name):
action_upgrade.append(app)
else:
all_in_one_upgrade.append(app)
run_all_in_one_upgrade(
all_in_one_upgrade,
new_source,
model_name=model_name)
run_action_upgrade(
action_upgrade,
new_source,
model_name=model_name)
def run_upgrade_tests(new_source, model_name=None):
"""Upgrade payload of all applications in model.
This the most basic upgrade test. It should be adapted to add/remove
elements from the environment and add tests at intermediate stages.
:param new_source: New package origin.
:type new_source: str
:param model_name: Name of model to query.
:type model_name: str
"""
groups = get_upgrade_groups(model_name=model_name)
run_upgrade(groups['Core Identity'], new_source, model_name=model_name)
run_upgrade(groups['Storage'], new_source, model_name=model_name)
run_upgrade(groups['Control Plane'], new_source, model_name=model_name)
run_upgrade(groups['Compute'], new_source, model_name=model_name)
run_upgrade(groups['sweep_up'], new_source, model_name=model_name)