arif/zaza-openstack-tests
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #197 from ChrisMacNaughton/feature/parallel-series-upgrade

Browse Source 
Enable parallel / asynchronous series upgrade
This commit is contained in:
Liam Young
2020-03-27 15:54:36 +00:00
committed by GitHub
parent b717690c2a d7f7134e58
commit f5feea4b51
10 changed files with 1633 additions and 697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
unit_tests/utilities/test_zaza_utilities_generic.py
View File

@@ -173,45 +173,6 @@ class TestGenericUtils(ut_utils.BaseTestCase):
_yaml_dict)
self._open.assert_called_once_with(_filename, "r")
def test_dist_upgrade(self):
_unit = "app/2"
generic_utils.dist_upgrade(_unit)
dist_upgrade_cmd = (
"""sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """
"""-o "Dpkg::Options::=--force-confdef" """
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
self.model.run_on_unit.assert_has_calls([
mock.call(_unit, 'sudo apt update'),
mock.call(_unit, dist_upgrade_cmd)])
def test_do_release_upgrade(self):
_unit = "app/2"
generic_utils.do_release_upgrade(_unit)
self.subprocess.check_call.assert_called_once_with(
['juju', 'ssh', _unit, 'sudo', 'DEBIAN_FRONTEND=noninteractive',
'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive'])
def test_wrap_do_release_upgrade(self):
self.patch_object(generic_utils, "do_release_upgrade")
self.patch_object(generic_utils, "run_via_ssh")
self.patch_object(generic_utils.model, "scp_to_unit")
_unit = "app/2"
_from_series = "xenial"
_to_series = "bionic"
_workaround_script = "scriptname"
_files = ["filename", _workaround_script]
_scp_calls = []
_run_calls = [
mock.call(_unit, _workaround_script)]
for filename in _files:
_scp_calls.append(mock.call(_unit, filename, filename))
generic_utils.wrap_do_release_upgrade(
_unit, to_series=_to_series, from_series=_from_series,
workaround_script=_workaround_script, files=_files)
self.scp_to_unit.assert_has_calls(_scp_calls)
self.run_via_ssh.assert_has_calls(_run_calls)
self.do_release_upgrade.assert_called_once_with(_unit)
def test_reboot(self):
_unit = "app/2"
generic_utils.reboot(_unit)
@@ -237,149 +198,6 @@ class TestGenericUtils(ut_utils.BaseTestCase):
self.set_application_config.assert_called_once_with(
_application, {_origin: _pocket})
def test_series_upgrade(self):
self.patch_object(generic_utils.model, "block_until_all_units_idle")
self.patch_object(generic_utils.model, "block_until_unit_wl_status")
self.patch_object(generic_utils.model, "prepare_series_upgrade")
self.patch_object(generic_utils.model, "complete_series_upgrade")
self.patch_object(generic_utils.model, "set_series")
self.patch_object(generic_utils, "set_origin")
self.patch_object(generic_utils, "wrap_do_release_upgrade")
self.patch_object(generic_utils, "reboot")
_unit = "app/2"
_application = "app"
_machine_num = "4"
_from_series = "xenial"
_to_series = "bionic"
_origin = "source"
_files = ["filename", "scriptname"]
_workaround_script = "scriptname"
generic_utils.series_upgrade(
_unit, _machine_num, origin=_origin,
to_series=_to_series, from_series=_from_series,
workaround_script=_workaround_script, files=_files)
self.block_until_all_units_idle.called_with()
self.prepare_series_upgrade.assert_called_once_with(
_machine_num, to_series=_to_series)
self.wrap_do_release_upgrade.assert_called_once_with(
_unit, to_series=_to_series, from_series=_from_series,
workaround_script=_workaround_script, files=_files)
self.complete_series_upgrade.assert_called_once_with(_machine_num)
self.set_series.assert_called_once_with(_application, _to_series)
self.set_origin.assert_called_once_with(_application, _origin)
self.reboot.assert_called_once_with(_unit)
def test_series_upgrade_application_pause_peers_and_subordinates(self):
self.patch_object(generic_utils.model, "run_action")
self.patch_object(generic_utils, "series_upgrade")
_application = "app"
_from_series = "xenial"
_to_series = "bionic"
_origin = "source"
_files = ["filename", "scriptname"]
_workaround_script = "scriptname"
_completed_machines = []
# Peers and Subordinates
_run_action_calls = [
mock.call("{}-hacluster/1".format(_application),
"pause", action_params={}),
mock.call("{}/1".format(_application), "pause", action_params={}),
mock.call("{}-hacluster/2".format(_application),
"pause", action_params={}),
mock.call("{}/2".format(_application), "pause", action_params={}),
]
_series_upgrade_calls = []
for machine_num in ("0", "1", "2"):
_series_upgrade_calls.append(
mock.call("{}/{}".format(_application, machine_num),
machine_num, origin=_origin,
from_series=_from_series, to_series=_to_series,
workaround_script=_workaround_script, files=_files,
post_upgrade_functions=None),
)
# Pause primary peers and subordinates
generic_utils.series_upgrade_application(
_application, origin=_origin,
to_series=_to_series, from_series=_from_series,
pause_non_leader_primary=True,
pause_non_leader_subordinate=True,
completed_machines=_completed_machines,
workaround_script=_workaround_script, files=_files),
self.run_action.assert_has_calls(_run_action_calls)
self.series_upgrade.assert_has_calls(_series_upgrade_calls)
def test_series_upgrade_application_pause_subordinates(self):
self.patch_object(generic_utils.model, "run_action")
self.patch_object(generic_utils, "series_upgrade")
_application = "app"
_from_series = "xenial"
_to_series = "bionic"
_origin = "source"
_files = ["filename", "scriptname"]
_workaround_script = "scriptname"
_completed_machines = []
# Subordinates only
_run_action_calls = [
mock.call("{}-hacluster/1".format(_application),
"pause", action_params={}),
mock.call("{}-hacluster/2".format(_application),
"pause", action_params={}),
]
_series_upgrade_calls = []
for machine_num in ("0", "1", "2"):
_series_upgrade_calls.append(
mock.call("{}/{}".format(_application, machine_num),
machine_num, origin=_origin,
from_series=_from_series, to_series=_to_series,
workaround_script=_workaround_script, files=_files,
post_upgrade_functions=None),
)
# Pause subordinates
generic_utils.series_upgrade_application(
_application, origin=_origin,
to_series=_to_series, from_series=_from_series,
pause_non_leader_primary=False,
pause_non_leader_subordinate=True,
completed_machines=_completed_machines,
workaround_script=_workaround_script, files=_files),
self.run_action.assert_has_calls(_run_action_calls)
self.series_upgrade.assert_has_calls(_series_upgrade_calls)
def test_series_upgrade_application_no_pause(self):
self.patch_object(generic_utils.model, "run_action")
self.patch_object(generic_utils, "series_upgrade")
_application = "app"
_from_series = "xenial"
_to_series = "bionic"
_origin = "source"
_series_upgrade_calls = []
_files = ["filename", "scriptname"]
_workaround_script = "scriptname"
_completed_machines = []
for machine_num in ("0", "1", "2"):
_series_upgrade_calls.append(
mock.call("{}/{}".format(_application, machine_num),
machine_num, origin=_origin,
from_series=_from_series, to_series=_to_series,
workaround_script=_workaround_script, files=_files,
post_upgrade_functions=None),
)
# No Pausiing
generic_utils.series_upgrade_application(
_application, origin=_origin,
to_series=_to_series, from_series=_from_series,
pause_non_leader_primary=False,
pause_non_leader_subordinate=False,
completed_machines=_completed_machines,
workaround_script=_workaround_script, files=_files)
self.run_action.assert_not_called()
self.series_upgrade.assert_has_calls(_series_upgrade_calls)
def test_set_dpkg_non_interactive_on_unit(self):
self.patch_object(generic_utils, "model")
_unit_name = "app/1"

33
unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py
View File

@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import mock
import unit_tests.utils as ut_utils
@@ -172,38 +171,6 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase):
{'source': 'new-src'},
model_name=None)
def test__extract_charm_name_from_url(self):
self.assertEqual(
openstack_upgrade._extract_charm_name_from_url(
'local:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade._extract_charm_name_from_url(
'cs:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade._extract_charm_name_from_url('cs:heat'),
'heat')
def test_get_upgrade_candidates(self):
expect = copy.deepcopy(self.juju_status.applications)
del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST
del expect['ntp'] # Filter as it has no source option
del expect['neutron-openvswitch'] # Filter as it is a subordinates
self.assertEqual(
openstack_upgrade.get_upgrade_candidates(),
expect)
def test_get_upgrade_groups(self):
self.assertEqual(
openstack_upgrade.get_upgrade_groups(),
{
'Compute': ['nova-compute'],
'Control Plane': ['cinder'],
'Core Identity': [],
'Storage': [],
'sweep_up': []})
def test_is_action_upgradable(self):
self.assertTrue(
openstack_upgrade.is_action_upgradable('cinder'))

239
unit_tests/utilities/test_zaza_utilities_series_upgrade.py Normal file
View File

@@ -0,0 +1,239 @@
# Copyright 2020 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unit_tests.utils as ut_utils
import zaza.openstack.utilities.generic as generic_utils
import zaza.openstack.utilities.series_upgrade as series_upgrade_utils
FAKE_STATUS = {
'can-upgrade-to': '',
'charm': 'local:trusty/app-136',
'subordinate-to': [],
'units': {'app/0': {'leader': True,
'machine': '0',
'subordinates': {
'app-hacluster/0': {
'charm': 'local:trusty/hacluster-0',
'leader': True}}},
'app/1': {'machine': '1',
'subordinates': {
'app-hacluster/1': {
'charm': 'local:trusty/hacluster-0'}}},
'app/2': {'machine': '2',
'subordinates': {
'app-hacluster/2': {
'charm': 'local:trusty/hacluster-0'}}}}}
class TestSeriesUpgrade(ut_utils.BaseTestCase):
def setUp(self):
super(TestSeriesUpgrade, self).setUp()
# Patch all subprocess calls
self.patch(
'zaza.openstack.utilities.generic.subprocess',
new_callable=mock.MagicMock(),
name='subprocess'
)
self.patch_object(generic_utils, "run_via_ssh")
# Juju Status Object and data
self.juju_status = mock.MagicMock()
self.juju_status.applications.__getitem__.return_value = FAKE_STATUS
self.patch_object(series_upgrade_utils, "model")
self.model.get_status.return_value = self.juju_status
def test_series_upgrade(self):
self.patch_object(
series_upgrade_utils.model, "block_until_all_units_idle")
self.patch_object(
series_upgrade_utils.model, "block_until_unit_wl_status")
self.patch_object(series_upgrade_utils.model, "prepare_series_upgrade")
self.patch_object(
series_upgrade_utils.model, "complete_series_upgrade")
self.patch_object(series_upgrade_utils.model, "set_series")
self.patch_object(generic_utils, "set_origin")
self.patch_object(series_upgrade_utils, "wrap_do_release_upgrade")
self.patch_object(generic_utils, "reboot")
_unit = "app/2"
_application = "app"
_machine_num = "4"
_from_series = "xenial"
_to_series = "bionic"
_origin = "source"
_files = ["filename", "scriptname"]
_workaround_script = "scriptname"
series_upgrade_utils.series_upgrade(
_unit, _machine_num, origin=_origin,
to_series=_to_series, from_series=_from_series,
workaround_script=_workaround_script, files=_files)
self.block_until_all_units_idle.called_with()
self.prepare_series_upgrade.assert_called_once_with(
_machine_num, to_series=_to_series)
self.wrap_do_release_upgrade.assert_called_once_with(
_unit, to_series=_to_series, from_series=_from_series,
workaround_script=_workaround_script, files=_files)
self.complete_series_upgrade.assert_called_once_with(_machine_num)
self.set_series.assert_called_once_with(_application, _to_series)
self.set_origin.assert_called_once_with(_application, _origin)
self.reboot.assert_called_once_with(_unit)
def test_series_upgrade_application_pause_peers_and_subordinates(self):
self.patch_object(series_upgrade_utils.model, "run_action")
self.patch_object(series_upgrade_utils, "series_upgrade")
_application = "app"
_from_series = "xenial"
_to_series = "bionic"
_origin = "source"
_files = ["filename", "scriptname"]
_workaround_script = "scriptname"
_completed_machines = []
# Peers and Subordinates
_run_action_calls = [
mock.call("{}-hacluster/1".format(_application),
"pause", action_params={}),
mock.call("{}/1".format(_application), "pause", action_params={}),
mock.call("{}-hacluster/2".format(_application),
"pause", action_params={}),
mock.call("{}/2".format(_application), "pause", action_params={}),
]
_series_upgrade_calls = []
for machine_num in ("0", "1", "2"):
_series_upgrade_calls.append(
mock.call("{}/{}".format(_application, machine_num),
machine_num, origin=_origin,
from_series=_from_series, to_series=_to_series,
workaround_script=_workaround_script, files=_files,
post_upgrade_functions=None),
)
# Pause primary peers and subordinates
series_upgrade_utils.series_upgrade_application(
_application, origin=_origin,
to_series=_to_series, from_series=_from_series,
pause_non_leader_primary=True,
pause_non_leader_subordinate=True,
completed_machines=_completed_machines,
workaround_script=_workaround_script, files=_files),
self.run_action.assert_has_calls(_run_action_calls)
self.series_upgrade.assert_has_calls(_series_upgrade_calls)
def test_series_upgrade_application_pause_subordinates(self):
self.patch_object(series_upgrade_utils.model, "run_action")
self.patch_object(series_upgrade_utils, "series_upgrade")
_application = "app"
_from_series = "xenial"
_to_series = "bionic"
_origin = "source"
_files = ["filename", "scriptname"]
_workaround_script = "scriptname"
_completed_machines = []
# Subordinates only
_run_action_calls = [
mock.call("{}-hacluster/1".format(_application),
"pause", action_params={}),
mock.call("{}-hacluster/2".format(_application),
"pause", action_params={}),
]
_series_upgrade_calls = []
for machine_num in ("0", "1", "2"):
_series_upgrade_calls.append(
mock.call("{}/{}".format(_application, machine_num),
machine_num, origin=_origin,
from_series=_from_series, to_series=_to_series,
workaround_script=_workaround_script, files=_files,
post_upgrade_functions=None),
)
# Pause subordinates
series_upgrade_utils.series_upgrade_application(
_application, origin=_origin,
to_series=_to_series, from_series=_from_series,
pause_non_leader_primary=False,
pause_non_leader_subordinate=True,
completed_machines=_completed_machines,
workaround_script=_workaround_script, files=_files),
self.run_action.assert_has_calls(_run_action_calls)
self.series_upgrade.assert_has_calls(_series_upgrade_calls)
def test_series_upgrade_application_no_pause(self):
self.patch_object(series_upgrade_utils.model, "run_action")
self.patch_object(series_upgrade_utils, "series_upgrade")
_application = "app"
_from_series = "xenial"
_to_series = "bionic"
_origin = "source"
_series_upgrade_calls = []
_files = ["filename", "scriptname"]
_workaround_script = "scriptname"
_completed_machines = []
for machine_num in ("0", "1", "2"):
_series_upgrade_calls.append(
mock.call("{}/{}".format(_application, machine_num),
machine_num, origin=_origin,
from_series=_from_series, to_series=_to_series,
workaround_script=_workaround_script, files=_files,
post_upgrade_functions=None),
)
# No Pausiing
series_upgrade_utils.series_upgrade_application(
_application, origin=_origin,
to_series=_to_series, from_series=_from_series,
pause_non_leader_primary=False,
pause_non_leader_subordinate=False,
completed_machines=_completed_machines,
workaround_script=_workaround_script, files=_files)
self.run_action.assert_not_called()
self.series_upgrade.assert_has_calls(_series_upgrade_calls)
def test_dist_upgrade(self):
_unit = "app/2"
series_upgrade_utils.dist_upgrade(_unit)
dist_upgrade_cmd = (
"""sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """
"""-o "Dpkg::Options::=--force-confdef" """
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
self.model.run_on_unit.assert_has_calls([
mock.call(_unit, 'sudo apt update'),
mock.call(_unit, dist_upgrade_cmd)])
def test_do_release_upgrade(self):
_unit = "app/2"
series_upgrade_utils.do_release_upgrade(_unit)
self.run_via_ssh.assert_called_once_with(
_unit,
'DEBIAN_FRONTEND=noninteractive do-release-upgrade '
'-f DistUpgradeViewNonInteractive')
def test_wrap_do_release_upgrade(self):
self.patch_object(series_upgrade_utils, "do_release_upgrade")
self.patch_object(series_upgrade_utils.model, "scp_to_unit")
_unit = "app/2"
_from_series = "xenial"
_to_series = "bionic"
_workaround_script = "scriptname"
_files = ["filename", _workaround_script]
_scp_calls = []
_run_calls = [
mock.call(_unit, _workaround_script)]
for filename in _files:
_scp_calls.append(mock.call(_unit, filename, filename))
series_upgrade_utils.wrap_do_release_upgrade(
_unit, to_series=_to_series, from_series=_from_series,
workaround_script=_workaround_script, files=_files)
self.scp_to_unit.assert_has_calls(_scp_calls)
self.run_via_ssh.assert_has_calls(_run_calls)
self.do_release_upgrade.assert_called_once_with(_unit)

128
unit_tests/utilities/test_zaza_utilities_upgrade_utils.py Normal file
View File

@@ -0,0 +1,128 @@
# Copyright 2020 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import copy
import mock
import pprint
import unit_tests.utils as ut_utils
import zaza.openstack.utilities.upgrade_utils as openstack_upgrade
class TestUpgradeUtils(ut_utils.BaseTestCase):
def setUp(self):
super(TestUpgradeUtils, self).setUp()
self.patch_object(
openstack_upgrade.zaza.model,
"get_units")
self.juju_status = mock.MagicMock()
self.patch_object(
openstack_upgrade.zaza.model,
"get_status",
return_value=self.juju_status)
self.patch_object(
openstack_upgrade.zaza.model,
"get_application_config")
def _get_application_config(app, model_name=None):
app_config = {
'ceph-mon': {'verbose': True, 'source': 'old-src'},
'neutron-openvswitch': {'verbose': True},
'ntp': {'verbose': True},
'percona-cluster': {'verbose': True, 'source': 'old-src'},
'cinder': {
'verbose': True,
'openstack-origin': 'old-src',
'action-managed-upgrade': False},
'neutron-api': {
'verbose': True,
'openstack-origin': 'old-src',
'action-managed-upgrade': False},
'nova-compute': {
'verbose': True,
'openstack-origin': 'old-src',
'action-managed-upgrade': False},
}
return app_config[app]
self.get_application_config.side_effect = _get_application_config
self.juju_status.applications = {
'mydb': { # Filter as it is on UPGRADE_EXCLUDE_LIST
'charm': 'cs:percona-cluster'},
'neutron-openvswitch': { # Filter as it is a subordinates
'charm': 'cs:neutron-openvswitch',
'subordinate-to': 'nova-compute'},
'ntp': { # Filter as it has no source option
'charm': 'cs:ntp'},
'nova-compute': {
'charm': 'cs:nova-compute',
'units': {
'nova-compute/0': {
'subordinates': {
'neutron-openvswitch/2': {
'charm': 'cs:neutron-openvswitch-22'}}}}},
'cinder': {
'charm': 'cs:cinder-23',
'units': {
'cinder/1': {
'subordinates': {
'cinder-hacluster/0': {
'charm': 'cs:hacluster-42'},
'cinder-ceph/3': {
'charm': 'cs:cinder-ceph-2'}}}}}}
def test_get_upgrade_candidates(self):
expected = copy.deepcopy(self.juju_status.applications)
self.assertEqual(
openstack_upgrade.get_upgrade_candidates(),
expected)
def test_get_upgrade_groups(self):
expected = collections.OrderedDict([
('Core Identity', []),
('Control Plane', ['cinder']),
('Data Plane', ['nova-compute']),
('sweep_up', [])])
actual = openstack_upgrade.get_upgrade_groups()
pprint.pprint(expected)
pprint.pprint(actual)
self.assertEqual(
actual,
expected)
def test_get_series_upgrade_groups(self):
expected = collections.OrderedDict([
('Core Identity', []),
('Control Plane', ['cinder']),
('Data Plane', ['nova-compute']),
('sweep_up', ['mydb', 'ntp'])])
actual = openstack_upgrade.get_series_upgrade_groups()
pprint.pprint(expected)
pprint.pprint(actual)
self.assertEqual(
actual,
expected)
def test_extract_charm_name_from_url(self):
self.assertEqual(
openstack_upgrade.extract_charm_name_from_url(
'local:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade.extract_charm_name_from_url(
'cs:bionic/heat-12'),
'heat')
self.assertEqual(
openstack_upgrade.extract_charm_name_from_url('cs:heat'),
'heat')

194
zaza/openstack/charm_tests/series_upgrade/tests.py
View File

@@ -16,6 +16,7 @@
"""Define class for Series Upgrade."""
import asyncio
import logging
import os
import unittest
@@ -23,13 +24,30 @@ import unittest
from zaza import model
from zaza.openstack.utilities import (
cli as cli_utils,
generic as generic_utils,
series_upgrade as series_upgrade_utils,
upgrade_utils as upgrade_utils,
)
from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest
def _filter_easyrsa(app, app_config, model_name=None):
charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm'])
if "easyrsa" in charm_name:
logging.warn("Skipping series upgrade of easyrsa Bug #1850121")
return True
return False
def _filter_etcd(app, app_config, model_name=None):
charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm'])
if "etcd" in charm_name:
logging.warn("Skipping series upgrade of easyrsa Bug #1850124")
return True
return False
class SeriesUpgradeTest(unittest.TestCase):
"""Class to encapsulate Sereis Upgrade Tests."""
"""Class to encapsulate Series Upgrade Tests."""
@classmethod
def setUpClass(cls):
@@ -47,77 +65,35 @@ class SeriesUpgradeTest(unittest.TestCase):
applications = model.get_status().applications
completed_machines = []
for application in applications:
# Defaults
origin = "openstack-origin"
pause_non_leader_subordinate = True
pause_non_leader_primary = True
post_upgrade_functions = []
for application, app_details in applications:
# Skip subordinates
if applications[application]["subordinate-to"]:
if app_details["subordinate-to"]:
continue
if "easyrsa" in applications[application]["charm"]:
logging.warn("Skipping series upgrade of easyrsa Bug #1850121")
if "easyrsa" in app_details["charm"]:
logging.warn(
"Skipping series upgrade of easyrsa Bug #1850121")
continue
if "etcd" in applications[application]["charm"]:
logging.warn("Skipping series upgrade of easyrsa Bug #1850124")
if "etcd" in app_details["charm"]:
logging.warn(
"Skipping series upgrade of easyrsa Bug #1850124")
continue
if "percona-cluster" in applications[application]["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = True
if "rabbitmq-server" in applications[application]["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = False
if "nova-compute" in applications[application]["charm"]:
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "ceph" in applications[application]["charm"]:
origin = "source"
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "designate-bind" in applications[application]["charm"]:
origin = None
if "tempest" in applications[application]["charm"]:
origin = None
if "memcached" in applications[application]["charm"]:
origin = None
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "vault" in applications[application]["charm"]:
origin = None
pause_non_leader_primary = False
pause_non_leader_subordinate = True
post_upgrade_functions = [
('zaza.openstack.charm_tests.vault.setup.'
'mojo_unseal_by_unit')]
if "mongodb" in applications[application]["charm"]:
# Mongodb needs to run series upgrade
# on its secondaries first.
generic_utils.series_upgrade_non_leaders_first(
application,
from_series=self.from_series,
to_series=self.to_series,
completed_machines=completed_machines,
post_upgrade_functions=post_upgrade_functions)
continue
# The rest are likley APIs use defaults
generic_utils.series_upgrade_application(
charm_name = upgrade_utils.extract_charm_name_from_url(
app_details['charm'])
upgrade_config = series_upgrade_utils.app_config(
charm_name,
is_async=False)
upgrade_function = upgrade_config.pop('upgrade_function')
logging.warn("About to upgrade {}".format(application))
upgrade_function(
application,
pause_non_leader_primary=pause_non_leader_primary,
pause_non_leader_subordinate=pause_non_leader_subordinate,
**upgrade_config,
from_series=self.from_series,
to_series=self.to_series,
origin=origin,
completed_machines=completed_machines,
workaround_script=self.workaround_script,
files=self.files,
post_upgrade_functions=post_upgrade_functions)
if "rabbitmq-server" in applications[application]["charm"]:
)
if "rabbitmq-server" in app_details["charm"]:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
@@ -126,7 +102,7 @@ class SeriesUpgradeTest(unittest.TestCase):
action_params={})
model.block_until_all_units_idle()
if "percona-cluster" in applications[application]["charm"]:
if "percona-cluster" in app_details["charm"]:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
@@ -215,5 +191,95 @@ class XenialBionicSeriesUpgrade(SeriesUpgradeTest):
cls.to_series = "bionic"
class ParallelSeriesUpgradeTest(unittest.TestCase):
"""Class to encapsulate Series Upgrade Tests."""
@classmethod
def setUpClass(cls):
"""Run setup for Series Upgrades."""
cli_utils.setup_logging()
cls.from_series = None
cls.to_series = None
cls.workaround_script = None
cls.files = []
def test_200_run_series_upgrade(self):
"""Run series upgrade."""
# Set Feature Flag
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
upgrade_groups = upgrade_utils.get_series_upgrade_groups(
extra_filters=[_filter_etcd, _filter_easyrsa])
applications = model.get_status().applications
completed_machines = []
for group_name, group in upgrade_groups.items():
logging.warn("About to upgrade {} ({})".format(group_name, group))
upgrade_group = []
for application, app_details in applications.items():
if application not in group:
continue
charm_name = upgrade_utils.extract_charm_name_from_url(
app_details['charm'])
upgrade_config = series_upgrade_utils.app_config(charm_name)
upgrade_function = upgrade_config.pop('upgrade_function')
logging.warn("About to upgrade {}".format(application))
upgrade_group.append(
upgrade_function(
application,
**upgrade_config,
from_series=self.from_series,
to_series=self.to_series,
completed_machines=completed_machines,
workaround_script=self.workaround_script,
files=self.files,
))
asyncio.get_event_loop().run_until_complete(
asyncio.gather(*upgrade_group))
if "rabbitmq-server" in group:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
'rabbitmq-server',
'complete-cluster-series-upgrade',
action_params={})
model.block_until_all_units_idle()
if "percona-cluster" in group:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
'mysql',
'complete-cluster-series-upgrade',
action_params={})
model.block_until_all_units_idle()
class ParallelTrustyXenialSeriesUpgrade(ParallelSeriesUpgradeTest):
"""Trusty to Xenial Series Upgrade.
Makes no assumptions about what is in the deployment.
"""
@classmethod
def setUpClass(cls):
"""Run setup for Trusty to Xenial Series Upgrades."""
super(ParallelTrustyXenialSeriesUpgrade, cls).setUpClass()
cls.from_series = "trusty"
cls.to_series = "xenial"
class ParallelXenialBionicSeriesUpgrade(ParallelSeriesUpgradeTest):
"""Xenial to Bionic Series Upgrade.
Makes no assumptions about what is in the deployment.
"""
@classmethod
def setUpClass(cls):
"""Run setup for Xenial to Bionic Series Upgrades."""
super(ParallelXenialBionicSeriesUpgrade, cls).setUpClass()
cls.from_series = "xenial"
cls.to_series = "bionic"
if __name__ == "__main__":
unittest.main()

15
zaza/openstack/utilities/charm_upgrade.py Normal file
View File

@@ -0,0 +1,15 @@
# Copyright 2020 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Collection of functions to support charm upgrade testing."""

400
zaza/openstack/utilities/generic.py
View File

@@ -14,6 +14,7 @@
"""Collection of functions that did not fit anywhere else."""
import asyncio
import logging
import os
import socket
@@ -25,11 +26,6 @@ from zaza import model
from zaza.openstack.utilities import juju as juju_utils
from zaza.openstack.utilities import exceptions as zaza_exceptions
from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE
from zaza.charm_lifecycle import utils as cl_utils
SUBORDINATE_PAUSE_RESUME_BLACKLIST = [
"cinder-ceph",
]
def dict_to_yaml(dict_data):
@@ -192,255 +188,6 @@ def get_yaml_config(config_file):
return yaml.safe_load(open(config_file, 'r').read())
def run_post_upgrade_functions(post_upgrade_functions):
"""Execute list supplied functions.
:param post_upgrade_functions: List of functions
:type post_upgrade_functions: [function, function, ...]
"""
if post_upgrade_functions:
for func in post_upgrade_functions:
logging.info("Running {}".format(func))
cl_utils.get_class(func)()
def series_upgrade_non_leaders_first(application, from_series="trusty",
to_series="xenial",
completed_machines=[],
post_upgrade_functions=None):
"""Series upgrade non leaders first.
Wrap all the functionality to handle series upgrade for charms
which must have non leaders upgraded first.
:param application: Name of application to upgrade series
:type application: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:returns: None
:rtype: None
"""
status = model.get_status().applications[application]
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Series upgrade the non-leaders first
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
series_upgrade(unit, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
run_post_upgrade_functions(post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
.format(unit, machine, application))
model.block_until_all_units_idle()
# Series upgrade the leader
machine = status["units"][leader]["machine"]
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
series_upgrade(leader, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
.format(unit, machine, application))
model.block_until_all_units_idle()
def series_upgrade_application(application, pause_non_leader_primary=True,
pause_non_leader_subordinate=True,
from_series="trusty", to_series="xenial",
origin='openstack-origin',
completed_machines=[],
files=None, workaround_script=None,
post_upgrade_functions=None):
"""Series upgrade application.
Wrap all the functionality to handle series upgrade for a given
application. Including pausing non-leader units.
:param application: Name of application to upgrade series
:type application: str
:param pause_non_leader_primary: Whether the non-leader applications should
be paused
:type pause_non_leader_primary: bool
:param pause_non_leader_subordinate: Whether the non-leader subordinate
hacluster applications should be
paused
:type pause_non_leader_subordinate: bool
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
status = model.get_status().applications[application]
# For some applications (percona-cluster) the leader unit must upgrade
# first. For API applications the non-leader haclusters must be paused
# before upgrade. Finally, for some applications this is arbitrary but
# generalized.
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Pause the non-leaders
for unit in non_leaders:
if pause_non_leader_subordinate:
if status["units"][unit].get("subordinates"):
for subordinate in status["units"][unit]["subordinates"]:
_app = subordinate.split('/')[0]
if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST:
logging.info("Skipping pausing {} - blacklisted"
.format(subordinate))
else:
logging.info("Pausing {}".format(subordinate))
model.run_action(
subordinate, "pause", action_params={})
if pause_non_leader_primary:
logging.info("Pausing {}".format(unit))
model.run_action(unit, "pause", action_params={})
machine = status["units"][leader]["machine"]
# Series upgrade the leader
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
series_upgrade(leader, machine,
from_series=from_series, to_series=to_series,
origin=origin, workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
set_origin(application, origin)
model.block_until_all_units_idle()
# Series upgrade the non-leaders
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
series_upgrade(unit, machine,
from_series=from_series, to_series=to_series,
origin=origin, workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
set_origin(application, origin)
model.block_until_all_units_idle()
def series_upgrade(unit_name, machine_num,
from_series="trusty", to_series="xenial",
origin='openstack-origin',
files=None, workaround_script=None,
post_upgrade_functions=None):
"""Perform series upgrade on a unit.
:param unit_name: Unit Name
:type unit_name: str
:param machine_num: Machine number
:type machine_num: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
logging.info("Series upgrade {}".format(unit_name))
application = unit_name.split('/')[0]
set_dpkg_non_interactive_on_unit(unit_name)
dist_upgrade(unit_name)
model.block_until_all_units_idle()
logging.info("Prepare series upgrade on {}".format(machine_num))
model.prepare_series_upgrade(machine_num, to_series=to_series)
logging.info("Waiting for workload status 'blocked' on {}"
.format(unit_name))
model.block_until_unit_wl_status(unit_name, "blocked")
logging.info("Waiting for model idleness")
model.block_until_all_units_idle()
wrap_do_release_upgrade(unit_name, from_series=from_series,
to_series=to_series, files=files,
workaround_script=workaround_script)
logging.info("Reboot {}".format(unit_name))
reboot(unit_name)
logging.info("Waiting for workload status 'blocked' on {}"
.format(unit_name))
model.block_until_unit_wl_status(unit_name, "blocked")
logging.info("Waiting for model idleness")
model.block_until_all_units_idle()
logging.info("Set origin on {}".format(application))
# Allow for charms which have neither source nor openstack-origin
if origin:
set_origin(application, origin)
model.block_until_all_units_idle()
logging.info("Complete series upgrade on {}".format(machine_num))
model.complete_series_upgrade(machine_num)
model.block_until_all_units_idle()
logging.info("Running run_post_upgrade_functions {}".format(
post_upgrade_functions))
run_post_upgrade_functions(post_upgrade_functions)
logging.info("Waiting for workload status 'active' on {}"
.format(unit_name))
model.block_until_unit_wl_status(unit_name, "active")
model.block_until_all_units_idle()
# This step may be performed by juju in the future
logging.info("Set series on {} to {}".format(application, to_series))
model.set_series(application, to_series)
def set_origin(application, origin='openstack-origin', pocket='distro'):
"""Set the configuration option for origin source.
@@ -459,44 +206,23 @@ def set_origin(application, origin='openstack-origin', pocket='distro'):
model.set_application_config(application, {origin: pocket})
def wrap_do_release_upgrade(unit_name, from_series="trusty",
to_series="xenial",
files=None, workaround_script=None):
"""Wrap do release upgrade.
async def async_set_origin(application, origin='openstack-origin',
pocket='distro'):
"""Set the configuration option for origin source.
In a production environment this step would be run administratively.
For testing purposes we need this automated.
:param unit_name: Unit Name
:type unit_name: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:param application: Name of application to upgrade series
:type application: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param pocket: Origin source cloud pocket.
i.e. 'distro' or 'cloud:xenial-newton'
:type pocket: str
:returns: None
:rtype: None
"""
# Pre upgrade hacks
# There are a few necessary hacks to accomplish an automated upgrade
# to overcome some packaging bugs.
# Copy scripts
if files:
logging.info("SCP files")
for _file in files:
logging.info("SCP {}".format(_file))
model.scp_to_unit(unit_name, _file, os.path.basename(_file))
# Run Script
if workaround_script:
logging.info("Running workaround script")
run_via_ssh(unit_name, workaround_script)
# Actually do the do_release_upgrade
do_release_upgrade(unit_name)
logging.info("Set origin on {} to {}".format(application, origin))
await model.async_set_application_config(application, {origin: pocket})
def run_via_ssh(unit_name, cmd):
@@ -521,24 +247,28 @@ def run_via_ssh(unit_name, cmd):
logging.warn(e)
def dist_upgrade(unit_name):
"""Run dist-upgrade on unit after update package db.
async def async_run_via_ssh(unit_name, cmd, raise_exceptions=False):
"""Run command on unit via ssh.
For executing commands on units when the juju agent is down.
:param unit_name: Unit Name
:type unit_name: str
:param cmd: Command to execute on remote unit
:type cmd: str
:returns: None
:rtype: None
"""
logging.info('Updating package db ' + unit_name)
update_cmd = 'sudo apt update'
model.run_on_unit(unit_name, update_cmd)
logging.info('Updating existing packages ' + unit_name)
dist_upgrade_cmd = (
"""sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """
"""-o "Dpkg::Options::=--force-confdef" """
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
model.run_on_unit(unit_name, dist_upgrade_cmd)
if "sudo" not in cmd:
# cmd.insert(0, "sudo")
cmd = "sudo {}".format(cmd)
cmd = ['juju', 'ssh', unit_name, cmd]
try:
await check_call(cmd)
except subprocess.CalledProcessError as e:
logging.warn("Failed command {} on {}".format(cmd, unit_name))
logging.warn(e)
if raise_exceptions:
raise e
def check_commands_on_units(commands, units):
@@ -566,26 +296,6 @@ def check_commands_on_units(commands, units):
return None
def do_release_upgrade(unit_name):
"""Run do-release-upgrade noninteractive.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
logging.info('Upgrading ' + unit_name)
# NOTE: It is necessary to run this via juju ssh rather than juju run due
# to timeout restrictions and error handling.
cmd = ['juju', 'ssh', unit_name, 'sudo', 'DEBIAN_FRONTEND=noninteractive',
'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive']
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
logging.warn("Failed do-release-upgrade for {}".format(unit_name))
logging.warn(e)
def reboot(unit_name):
"""Reboot unit.
@@ -604,6 +314,38 @@ def reboot(unit_name):
pass
async def async_reboot(unit_name):
"""Reboot unit.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
# NOTE: When used with series upgrade the agent will be down.
# Even juju run will not work
await async_run_via_ssh(unit_name, "sudo reboot && exit")
async def check_call(cmd):
"""Asynchronous function to check a subprocess call.
:param cmd: Command to execute
:type cmd: List[str]
:returns: None
:rtype: None
"""
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
logging.warn("STDOUT: {}".format(stdout))
logging.warn("STDERR: {}".format(stderr))
raise subprocess.CalledProcessError(proc.returncode, cmd)
def set_dpkg_non_interactive_on_unit(
unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"):
"""Set dpkg options on unit.
@@ -620,6 +362,22 @@ def set_dpkg_non_interactive_on_unit(
model.run_on_unit(unit_name, cmd)
async def async_set_dpkg_non_interactive_on_unit(
unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"):
"""Set dpkg options on unit.
:param unit_name: Unit Name
:type unit_name: str
:param apt_conf_d: Apt.conf file to update
:type apt_conf_d: str
"""
DPKG_NON_INTERACTIVE = 'DPkg::options { "--force-confdef"; };'
# Check if the option exists. If not, add it to the apt.conf.d file
cmd = ("grep '{option}' {file_name} || echo '{option}' >> {file_name}"
.format(option=DPKG_NON_INTERACTIVE, file_name=apt_conf_d))
await model.async_run_on_unit(unit_name, cmd)
def get_process_id_list(unit_name, process_name,
expect_success=True):
"""Get a list of process ID(s).

100
zaza/openstack/utilities/openstack_upgrade.py
View File

@@ -16,26 +16,14 @@
This module contains a number of functions for upgrading OpenStack.
"""
import re
import logging
import zaza.openstack.utilities.juju as juju_utils
import zaza.model
from zaza import sync_wrapper
SERVICE_GROUPS = {
'Core Identity': ['keystone'],
'Storage': [
'ceph-mon', 'ceph-osd', 'ceph-fs', 'ceph-radosgw', 'swift-proxy',
'swift-storage'],
'Control Plane': [
'aodh', 'barbican', 'ceilometer', 'cinder', 'designate',
'designate-bind', 'glance', 'gnocchi', 'heat', 'manila',
'manila-generic', 'neutron-api', 'neutron-gateway', 'placement',
'nova-cloud-controller', 'openstack-dashboard'],
'Compute': ['nova-compute']}
UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster']
from zaza.openstack.utilities.upgrade_utils import (
get_upgrade_groups,
)
async def async_pause_units(units, model_name=None):
@@ -189,88 +177,6 @@ def set_upgrade_application_config(applications, new_source,
model_name=model_name)
def _extract_charm_name_from_url(charm_url):
"""Extract the charm name from the charm url.
E.g. Extract 'heat' from local:bionic/heat-12
:param charm_url: Name of model to query.
:type charm_url: str
:returns: Charm name
:rtype: str
"""
charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1])
return charm_name.split(':')[-1]
def get_upgrade_candidates(model_name=None):
"""Extract list of apps from model that can be upgraded.
:param model_name: Name of model to query.
:type model_name: str
:returns: List of application that can have their payload upgraded.
:rtype: []
"""
status = zaza.model.get_status(model_name=model_name)
candidates = {}
for app, app_config in status.applications.items():
# Filter out subordinates
if app_config.get("subordinate-to"):
logging.warning(
"Excluding {} from upgrade, it is a subordinate".format(app))
continue
# Filter out charms on the naughty list
charm_name = _extract_charm_name_from_url(app_config['charm'])
if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST:
logging.warning(
"Excluding {} from upgrade, on the exclude list".format(app))
continue
# Filter out charms that have no source option
charm_options = zaza.model.get_application_config(
app, model_name=model_name).keys()
src_options = ['openstack-origin', 'source']
if not [x for x in src_options if x in charm_options]:
logging.warning(
"Excluding {} from upgrade, no src option".format(app))
continue
candidates[app] = app_config
return candidates
def get_upgrade_groups(model_name=None):
"""Place apps in the model into their upgrade groups.
Place apps in the model into their upgrade groups. If an app is deployed
but is not in SERVICE_GROUPS then it is placed in a sweep_up group.
:param model_name: Name of model to query.
:type model_name: str
:returns: Dict of group lists keyed on group name.
:rtype: {}
"""
apps_in_model = get_upgrade_candidates(model_name=model_name)
groups = {}
for phase_name, charms in SERVICE_GROUPS.items():
group = []
for app, app_config in apps_in_model.items():
charm_name = _extract_charm_name_from_url(app_config['charm'])
if charm_name in charms:
group.append(app)
groups[phase_name] = group
sweep_up = []
for app in apps_in_model:
if not (app in [a for group in groups.values() for a in group]):
sweep_up.append(app)
groups['sweep_up'] = sweep_up
return groups
def is_action_upgradable(app, model_name=None):
"""Can application be upgraded using action managed upgrade method.

855
zaza/openstack/utilities/series_upgrade.py Normal file
View File

@@ -0,0 +1,855 @@
# Copyright 2020 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Collection of functions for testing series upgrade."""
import collections
import copy
import concurrent
import logging
import os
from zaza import model
from zaza.charm_lifecycle import utils as cl_utils
import zaza.openstack.utilities.generic as os_utils
SUBORDINATE_PAUSE_RESUME_BLACKLIST = [
"cinder-ceph",
]
def app_config(charm_name, is_async=True):
"""Return a dict with the upgrade config for an application.
:param charm_name: Name of the charm about to upgrade
:type charm_name: str
:param async: Whether the upgreade functions should be async
:type async: bool
:returns: A dicitonary of the upgrade config for the application
:rtype: Dict
"""
if is_async:
default_upgrade = async_series_upgrade_application
secondary_first_upgrade = async_series_upgrade_non_leaders_first
else:
default_upgrade = series_upgrade_application
secondary_first_upgrade = series_upgrade_non_leaders_first
default = {
'origin': 'openstack-origin',
'pause_non_leader_subordinate': True,
'pause_non_leader_primary': True,
'upgrade_function': default_upgrade,
'post_upgrade_functions': []}
_app_settings = collections.defaultdict(lambda: default)
ceph = {
'origin': "source",
'pause_non_leader_primary': False,
'pause_non_leader_subordinate': False,
}
exceptions = {
'rabbitmq-server': {
'origin': 'source',
'pause_non_leader_subordinate': False, },
'percona-cluster': {'origin': 'source', },
'nova-compute': {
'pause_non_leader_primary': False,
'pause_non_leader_subordinate': False, },
'ceph': ceph,
'ceph-mon': ceph,
'ceph-osd': ceph,
'designate-bind': {'origin': None, },
'tempest': {'origin': None, },
'memcached': {
'origin': None,
'pause_non_leader_primary': False,
'pause_non_leader_subordinate': False,
},
'vault': {
'origin': None,
'pause_non_leader_primary': False,
'pause_non_leader_subordinate': True,
'post_upgrade_functions': [
('zaza.openstack.charm_tests.vault.setup.'
'mojo_unseal_by_unit')]
},
'mongodb': {
'upgrade_function': secondary_first_upgrade,
}
}
for key, value in exceptions.items():
_app_settings[key] = copy.deepcopy(default)
_app_settings[key].update(value)
return _app_settings[charm_name]
def run_post_upgrade_functions(post_upgrade_functions):
"""Execute list supplied functions.
:param post_upgrade_functions: List of functions
:type post_upgrade_functions: [function, function, ...]
"""
if post_upgrade_functions:
for func in post_upgrade_functions:
logging.info("Running {}".format(func))
cl_utils.get_class(func)()
def series_upgrade_non_leaders_first(application, from_series="trusty",
to_series="xenial",
completed_machines=[],
post_upgrade_functions=None):
"""Series upgrade non leaders first.
Wrap all the functionality to handle series upgrade for charms
which must have non leaders upgraded first.
:param application: Name of application to upgrade series
:type application: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:returns: None
:rtype: None
"""
status = model.get_status().applications[application]
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Series upgrade the non-leaders first
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
series_upgrade(unit, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
run_post_upgrade_functions(post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
.format(unit, machine, application))
model.block_until_all_units_idle()
# Series upgrade the leader
machine = status["units"][leader]["machine"]
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
series_upgrade(leader, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
.format(unit, machine, application))
model.block_until_all_units_idle()
async def async_series_upgrade_non_leaders_first(application,
from_series="trusty",
to_series="xenial",
completed_machines=[],
post_upgrade_functions=None):
"""Series upgrade non leaders first.
Wrap all the functionality to handle series upgrade for charms
which must have non leaders upgraded first.
:param application: Name of application to upgrade series
:type application: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:returns: None
:rtype: None
"""
status = (await model.async_get_status()).applications[application]
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Series upgrade the non-leaders first
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
await async_series_upgrade(
unit, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
run_post_upgrade_functions(post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
.format(unit, machine, application))
await model.async_block_until_all_units_idle()
# Series upgrade the leader
machine = status["units"][leader]["machine"]
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
await async_series_upgrade(
leader, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
.format(unit, machine, application))
await model.async_block_until_all_units_idle()
def series_upgrade_application(application, pause_non_leader_primary=True,
pause_non_leader_subordinate=True,
from_series="trusty", to_series="xenial",
origin='openstack-origin',
completed_machines=[],
files=None, workaround_script=None,
post_upgrade_functions=None):
"""Series upgrade application.
Wrap all the functionality to handle series upgrade for a given
application. Including pausing non-leader units.
:param application: Name of application to upgrade series
:type application: str
:param pause_non_leader_primary: Whether the non-leader applications should
be paused
:type pause_non_leader_primary: bool
:param pause_non_leader_subordinate: Whether the non-leader subordinate
hacluster applications should be
paused
:type pause_non_leader_subordinate: bool
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
status = model.get_status().applications[application]
# For some applications (percona-cluster) the leader unit must upgrade
# first. For API applications the non-leader haclusters must be paused
# before upgrade. Finally, for some applications this is arbitrary but
# generalized.
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Pause the non-leaders
for unit in non_leaders:
if pause_non_leader_subordinate:
if status["units"][unit].get("subordinates"):
for subordinate in status["units"][unit]["subordinates"]:
_app = subordinate.split('/')[0]
if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST:
logging.info("Skipping pausing {} - blacklisted"
.format(subordinate))
else:
logging.info("Pausing {}".format(subordinate))
model.run_action(
subordinate, "pause", action_params={})
if pause_non_leader_primary:
logging.info("Pausing {}".format(unit))
model.run_action(unit, "pause", action_params={})
machine = status["units"][leader]["machine"]
# Series upgrade the leader
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
series_upgrade(leader, machine,
from_series=from_series, to_series=to_series,
origin=origin, workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
os_utils.set_origin(application, origin)
model.block_until_all_units_idle()
# Series upgrade the non-leaders
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
series_upgrade(unit, machine,
from_series=from_series, to_series=to_series,
origin=origin, workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
os_utils.set_origin(application, origin)
model.block_until_all_units_idle()
async def async_series_upgrade_application(
application,
pause_non_leader_primary=True,
pause_non_leader_subordinate=True,
from_series="trusty",
to_series="xenial",
origin='openstack-origin',
completed_machines=None,
files=None, workaround_script=None,
post_upgrade_functions=None,
post_application_upgrade_functions=None):
"""Series upgrade application.
Wrap all the functionality to handle series upgrade for a given
application. Including pausing non-leader units.
:param application: Name of application to upgrade series
:type application: str
:param pause_non_leader_primary: Whether the non-leader applications should
be paused
:type pause_non_leader_primary: bool
:param pause_non_leader_subordinate: Whether the non-leader subordinate
hacluster applications should be
paused
:type pause_non_leader_subordinate: bool
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:param post_upgrade_functions: A list of functions to call after upgrading
each unit of an application
:type post_upgrade_functions: List[fn]
:param post_application_upgrade_functions: A list of functions to call
once after updating all units
of an application
:type post_application_upgrade_functions: List[fn]
:returns: None
:rtype: None
"""
if completed_machines is None:
completed_machines = []
status = (await model.async_get_status()).applications[application]
# For some applications (percona-cluster) the leader unit must upgrade
# first. For API applications the non-leader haclusters must be paused
# before upgrade. Finally, for some applications this is arbitrary but
# generalized.
leader = None
non_leaders = []
logging.info("Configuring leader / non leaders for {}".format(application))
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Pause the non-leaders
for unit in non_leaders:
if pause_non_leader_subordinate:
if status["units"][unit].get("subordinates"):
for subordinate in status["units"][unit]["subordinates"]:
_app = subordinate.split('/')[0]
if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST:
logging.info("Skipping pausing {} - blacklisted"
.format(subordinate))
else:
logging.info("Pausing {}".format(subordinate))
await model.async_run_action(
subordinate, "pause", action_params={})
if pause_non_leader_primary:
logging.info("Pausing {}".format(unit))
await model.async_run_action(unit, "pause", action_params={})
machine = status["units"][leader]["machine"]
# Series upgrade the leader
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
await async_series_upgrade(
leader, machine,
from_series=from_series,
to_series=to_series,
origin=origin,
workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
await os_utils.async_set_origin(application, origin)
await wait_for_unit_idle(unit)
# Series upgrade the non-leaders
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
await async_series_upgrade(
unit, machine,
from_series=from_series,
to_series=to_series,
origin=origin,
workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
await os_utils.async_set_origin(application, origin)
await wait_for_unit_idle(unit)
run_post_upgrade_functions(post_application_upgrade_functions)
# TODO: Move these functions into zaza.model
async def wait_for_unit_idle(unit_name, timeout=600):
"""Wait until the unit's agent is idle.
:param unit_name: The unit name of the application, ex: mysql/0
:type unit_name: str
:param timeout: How long to wait before timing out
:type timeout: int
:returns: None
:rtype: None
"""
app = unit_name.split('/')[0]
try:
await model.async_block_until(
_unit_idle(app, unit_name),
timeout=timeout)
except concurrent.futures._base.TimeoutError:
raise model.ModelTimeout("Zaza has timed out waiting on {} to "
"reach idle state.".format(unit_name))
def _unit_idle(app, unit_name):
async def f():
x = await get_agent_status(app, unit_name)
return x == "idle"
return f
async def get_agent_status(app, unit_name):
"""Get the current status of the specified unit.
:param app: The name of the Juju application, ex: mysql
:type app: str
:param unit_name: The unit name of the application, ex: mysql/0
:type unit_name: str
:returns: The agent status, either active / idle, returned by Juju
:rtype: str
"""
return (await model.async_get_status()). \
applications[app]['units'][unit_name]['agent-status']['status']
def series_upgrade(unit_name, machine_num,
from_series="trusty", to_series="xenial",
origin='openstack-origin',
files=None, workaround_script=None,
post_upgrade_functions=None):
"""Perform series upgrade on a unit.
:param unit_name: Unit Name
:type unit_name: str
:param machine_num: Machine number
:type machine_num: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
logging.info("Series upgrade {}".format(unit_name))
application = unit_name.split('/')[0]
os_utils.set_dpkg_non_interactive_on_unit(unit_name)
dist_upgrade(unit_name)
model.block_until_all_units_idle()
logging.info("Prepare series upgrade on {}".format(machine_num))
model.prepare_series_upgrade(machine_num, to_series=to_series)
logging.info("Waiting for workload status 'blocked' on {}"
.format(unit_name))
model.block_until_unit_wl_status(unit_name, "blocked")
logging.info("Waiting for model idleness")
model.block_until_all_units_idle()
wrap_do_release_upgrade(unit_name, from_series=from_series,
to_series=to_series, files=files,
workaround_script=workaround_script)
logging.info("Reboot {}".format(unit_name))
os_utils.reboot(unit_name)
logging.info("Waiting for workload status 'blocked' on {}"
.format(unit_name))
model.block_until_unit_wl_status(unit_name, "blocked")
logging.info("Waiting for model idleness")
model.block_until_all_units_idle()
logging.info("Set origin on {}".format(application))
# Allow for charms which have neither source nor openstack-origin
if origin:
os_utils.set_origin(application, origin)
model.block_until_all_units_idle()
logging.info("Complete series upgrade on {}".format(machine_num))
model.complete_series_upgrade(machine_num)
model.block_until_all_units_idle()
logging.info("Running run_post_upgrade_functions {}".format(
post_upgrade_functions))
run_post_upgrade_functions(post_upgrade_functions)
logging.info("Waiting for workload status 'active' on {}"
.format(unit_name))
model.block_until_unit_wl_status(unit_name, "active")
model.block_until_all_units_idle()
# This step may be performed by juju in the future
logging.info("Set series on {} to {}".format(application, to_series))
model.set_series(application, to_series)
async def async_series_upgrade(unit_name, machine_num,
from_series="trusty", to_series="xenial",
origin='openstack-origin',
files=None, workaround_script=None,
post_upgrade_functions=None):
"""Perform series upgrade on a unit.
:param unit_name: Unit Name
:type unit_name: str
:param machine_num: Machine number
:type machine_num: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
logging.info("Series upgrade {}".format(unit_name))
application = unit_name.split('/')[0]
await os_utils.async_set_dpkg_non_interactive_on_unit(unit_name)
await async_dist_upgrade(unit_name)
await wait_for_unit_idle(unit_name)
logging.info("Prepare series upgrade on {}".format(machine_num))
await async_prepare_series_upgrade(machine_num, to_series=to_series)
logging.info("Waiting for workload status 'blocked' on {}"
.format(unit_name))
await model.async_block_until_unit_wl_status(unit_name, "blocked")
logging.info("Waiting for unit {} idleness".format(unit_name))
await wait_for_unit_idle(unit_name)
await async_wrap_do_release_upgrade(unit_name, from_series=from_series,
to_series=to_series, files=files,
workaround_script=workaround_script)
logging.info("Reboot {}".format(unit_name))
await os_utils.async_reboot(unit_name)
logging.info("Waiting for workload status 'blocked' on {}"
.format(unit_name))
await model.async_block_until_unit_wl_status(unit_name, "blocked")
# Allow for charms which have neither source nor openstack-origin
if origin:
logging.info("Set origin on {}".format(application))
await os_utils.async_set_origin(application, origin)
await wait_for_unit_idle(unit_name)
logging.info("Complete series upgrade on {}".format(machine_num))
await async_complete_series_upgrade(machine_num)
await wait_for_unit_idle(unit_name, timeout=1200)
logging.info("Running run_post_upgrade_functions {}".format(
post_upgrade_functions))
run_post_upgrade_functions(post_upgrade_functions)
logging.info("Waiting for workload status 'active' on {}"
.format(unit_name))
await model.async_block_until_unit_wl_status(unit_name, "active")
await wait_for_unit_idle(unit_name)
# This step may be performed by juju in the future
logging.info("Set series on {} to {}".format(application, to_series))
await async_set_series(application, to_series)
async def async_prepare_series_upgrade(machine_num, to_series="xenial"):
"""Execute juju series-upgrade prepare on machine.
NOTE: This is a new feature in juju behind a feature flag and not yet in
libjuju.
export JUJU_DEV_FEATURE_FLAGS=upgrade-series
:param machine_num: Machine number
:type machine_num: str
:param to_series: The series to which to upgrade
:type to_series: str
:returns: None
:rtype: None
"""
juju_model = await model.async_get_juju_model()
cmd = ["juju", "upgrade-series", "-m", juju_model,
machine_num, "prepare", to_series, "--yes"]
logging.info("About to call '{}'".format(cmd))
await os_utils.check_call(cmd)
async def async_complete_series_upgrade(machine_num):
"""Execute juju series-upgrade complete on machine.
NOTE: This is a new feature in juju behind a feature flag and not yet in
libjuju.
export JUJU_DEV_FEATURE_FLAGS=upgrade-series
:param machine_num: Machine number
:type machine_num: str
:returns: None
:rtype: None
"""
juju_model = await model.async_get_juju_model()
cmd = ["juju", "upgrade-series", "-m", juju_model,
machine_num, "complete"]
await os_utils.check_call(cmd)
async def async_set_series(application, to_series):
"""Execute juju set-series complete on application.
NOTE: This is a new feature in juju and not yet in libjuju.
:param application: Name of application to upgrade series
:type application: str
:param to_series: The series to which to upgrade
:type to_series: str
:returns: None
:rtype: None
"""
juju_model = await model.async_get_juju_model()
cmd = ["juju", "set-series", "-m", juju_model,
application, to_series]
await os_utils.check_call(cmd)
def wrap_do_release_upgrade(unit_name, from_series="trusty",
to_series="xenial",
files=None, workaround_script=None):
"""Wrap do release upgrade.
In a production environment this step would be run administratively.
For testing purposes we need this automated.
:param unit_name: Unit Name
:type unit_name: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
# Pre upgrade hacks
# There are a few necessary hacks to accomplish an automated upgrade
# to overcome some packaging bugs.
# Copy scripts
if files:
logging.info("SCP files")
for _file in files:
logging.info("SCP {}".format(_file))
model.scp_to_unit(unit_name, _file, os.path.basename(_file))
# Run Script
if workaround_script:
logging.info("Running workaround script")
os_utils.run_via_ssh(unit_name, workaround_script)
# Actually do the do_release_upgrade
do_release_upgrade(unit_name)
async def async_wrap_do_release_upgrade(unit_name, from_series="trusty",
to_series="xenial",
files=None, workaround_script=None):
"""Wrap do release upgrade.
In a production environment this step would be run administratively.
For testing purposes we need this automated.
:param unit_name: Unit Name
:type unit_name: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
# Pre upgrade hacks
# There are a few necessary hacks to accomplish an automated upgrade
# to overcome some packaging bugs.
# Copy scripts
if files:
logging.info("SCP files")
for _file in files:
logging.info("SCP {}".format(_file))
await model.async_scp_to_unit(
unit_name, _file, os.path.basename(_file))
# Run Script
if workaround_script:
logging.info("Running workaround script")
await os_utils.async_run_via_ssh(unit_name, workaround_script)
# Actually do the do_release_upgrade
await async_do_release_upgrade(unit_name)
def dist_upgrade(unit_name):
"""Run dist-upgrade on unit after update package db.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
logging.info('Updating package db ' + unit_name)
update_cmd = 'sudo apt update'
model.run_on_unit(unit_name, update_cmd)
logging.info('Updating existing packages ' + unit_name)
dist_upgrade_cmd = (
"""sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """
"""-o "Dpkg::Options::=--force-confdef" """
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
model.run_on_unit(unit_name, dist_upgrade_cmd)
async def async_dist_upgrade(unit_name):
"""Run dist-upgrade on unit after update package db.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
logging.info('Updating package db ' + unit_name)
update_cmd = 'sudo apt update'
await model.async_run_on_unit(unit_name, update_cmd)
logging.info('Updating existing packages ' + unit_name)
dist_upgrade_cmd = (
"""sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """
"""-o "Dpkg::Options::=--force-confdef" """
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
await model.async_run_on_unit(unit_name, dist_upgrade_cmd)
def do_release_upgrade(unit_name):
"""Run do-release-upgrade noninteractive.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
logging.info('Upgrading ' + unit_name)
# NOTE: It is necessary to run this via juju ssh rather than juju run due
# to timeout restrictions and error handling.
os_utils.run_via_ssh(
unit_name,
'DEBIAN_FRONTEND=noninteractive '
'do-release-upgrade -f DistUpgradeViewNonInteractive')
async def async_do_release_upgrade(unit_name):
"""Run do-release-upgrade noninteractive.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
logging.info('Upgrading ' + unit_name)
# NOTE: It is necessary to run this via juju ssh rather than juju run due
# to timeout restrictions and error handling.
await os_utils.async_run_via_ssh(
unit_name,
'DEBIAN_FRONTEND=noninteractive '
'do-release-upgrade -f DistUpgradeViewNonInteractive',
raise_exceptions=True)

184
zaza/openstack/utilities/upgrade_utils.py Normal file
View File

@@ -0,0 +1,184 @@
# Copyright 2020 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Collection of functions to support upgrade testing."""
import re
import logging
import collections
import zaza.model
SERVICE_GROUPS = collections.OrderedDict([
('Core Identity', ['keystone']),
('Control Plane', [
'aodh', 'barbican', 'ceilometer', 'ceph-mon', 'ceph-fs',
'ceph-radosgw', 'cinder', 'designate',
'designate-bind', 'glance', 'gnocchi', 'heat', 'manila',
'manila-generic', 'neutron-api', 'neutron-gateway', 'placement',
'nova-cloud-controller', 'openstack-dashboard']),
('Data Plane', [
'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage'])
])
UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster']
def get_upgrade_candidates(model_name=None, filters=None):
"""Extract list of apps from model that can be upgraded.
:param model_name: Name of model to query.
:type model_name: str
:param filters: List of filter functions to apply
:type filters: List[fn]
:returns: List of application that can have their payload upgraded.
:rtype: []
"""
if filters is None:
filters = []
status = zaza.model.get_status(model_name=model_name)
candidates = {}
for app, app_config in status.applications.items():
if _include_app(app, app_config, filters, model_name=model_name):
candidates[app] = app_config
return candidates
def _include_app(app, app_config, filters, model_name=None):
for filt in filters:
if filt(app, app_config, model_name=model_name):
return False
return True
def _filter_subordinates(app, app_config, model_name=None):
if app_config.get("subordinate-to"):
logging.warning(
"Excluding {} from upgrade, it is a subordinate".format(app))
return True
return False
def _filter_openstack_upgrade_list(app, app_config, model_name=None):
charm_name = extract_charm_name_from_url(app_config['charm'])
if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST:
print("Excluding {} from upgrade, on the exclude list".format(app))
logging.warning(
"Excluding {} from upgrade, on the exclude list".format(app))
return True
return False
def _filter_non_openstack_services(app, app_config, model_name=None):
charm_options = zaza.model.get_application_config(
app, model_name=model_name).keys()
src_options = ['openstack-origin', 'source']
if not [x for x in src_options if x in charm_options]:
logging.warning(
"Excluding {} from upgrade, no src option".format(app))
return True
return False
def get_upgrade_groups(model_name=None, extra_filters=None):
"""Place apps in the model into their upgrade groups.
Place apps in the model into their upgrade groups. If an app is deployed
but is not in SERVICE_GROUPS then it is placed in a sweep_up group.
:param model_name: Name of model to query.
:type model_name: str
:returns: Dict of group lists keyed on group name.
:rtype: collections.OrderedDict
"""
filters = [
_filter_subordinates,
_filter_openstack_upgrade_list,
_filter_non_openstack_services,
]
if extra_filters:
if isinstance(extra_filters, list):
filters.extend(extra_filters)
elif callable(extra_filters):
filters.append(extra_filters)
else:
raise RuntimeError(
"extra_filters should be a list of "
"callables")
apps_in_model = get_upgrade_candidates(
model_name=model_name,
filters=filters,)
return _build_service_groups(apps_in_model)
def get_series_upgrade_groups(model_name=None, extra_filters=None):
"""Place apps in the model into their upgrade groups.
Place apps in the model into their upgrade groups. If an app is deployed
but is not in SERVICE_GROUPS then it is placed in a sweep_up group.
:param model_name: Name of model to query.
:type model_name: str
:returns: Dict of group lists keyed on group name.
:rtype: collections.OrderedDict
"""
filters = [_filter_subordinates]
if extra_filters:
if isinstance(extra_filters, list):
filters.extend(extra_filters)
elif callable(extra_filters):
filters.append(extra_filters)
else:
raise RuntimeError(
"extra_filters should be a list of "
"callables")
apps_in_model = get_upgrade_candidates(
model_name=model_name,
filters=filters)
return _build_service_groups(apps_in_model)
def _build_service_groups(applications):
groups = collections.OrderedDict()
for phase_name, charms in SERVICE_GROUPS.items():
group = []
for app, app_config in applications.items():
charm_name = extract_charm_name_from_url(app_config['charm'])
if charm_name in charms:
group.append(app)
groups[phase_name] = group
sweep_up = []
for app in applications:
if not (app in [a for group in groups.values() for a in group]):
sweep_up.append(app)
groups['sweep_up'] = sweep_up
for name, group in groups.items():
group.sort()
return groups
def extract_charm_name_from_url(charm_url):
"""Extract the charm name from the charm url.
E.g. Extract 'heat' from local:bionic/heat-12
:param charm_url: Name of model to query.
:type charm_url: str
:returns: Charm name
:rtype: str
"""
charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1])
return charm_name.split(':')[-1]