Merge pull request #224 from ChrisMacNaughton/feature/parallel-series-upgrade-refactor
Series upgrade can be done in massively parallel batches
This commit is contained in:
Liam Young
GitHub
@@ -8,7 +8,7 @@ setenv = VIRTUAL_ENV={envdir}
|
||||
install_command =
|
||||
pip install {opts} {packages}
|
||||
|
||||
commands = nosetests --with-coverage --cover-package=zaza {posargs} {toxinidir}/unit_tests
|
||||
commands = nosetests --with-coverage --cover-package=zaza.openstack {posargs} {toxinidir}/unit_tests
|
||||
|
||||
[testenv:py3]
|
||||
basepython = python3
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Copyright 2020 Canonical Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
@@ -0,0 +1,40 @@
|
||||
# Copyright 2020 Canonical Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import mock
|
||||
import unittest
|
||||
import sys
|
||||
|
||||
import zaza.openstack.charm_tests.mysql.utils as mysql_utils
|
||||
|
||||
|
||||
class TestMysqlUtils(unittest.TestCase):
|
||||
"""Test class to encapsulate testing Mysql test utils."""
|
||||
|
||||
def setUp(self):
|
||||
super(TestMysqlUtils, self).setUp()
|
||||
if sys.version_info < (3, 6, 0):
|
||||
raise unittest.SkipTest("Can't AsyncMock in py35")
|
||||
|
||||
@mock.patch.object(mysql_utils, 'model')
|
||||
def test_mysql_complete_cluster_series_upgrade(self, mock_model):
|
||||
run_action_on_leader = mock.AsyncMock()
|
||||
mock_model.async_run_action_on_leader = run_action_on_leader
|
||||
asyncio.get_event_loop().run_until_complete(
|
||||
mysql_utils.complete_cluster_series_upgrade())
|
||||
run_action_on_leader.assert_called_once_with(
|
||||
'mysql',
|
||||
'complete-cluster-series-upgrade',
|
||||
action_params={})
|
||||
@@ -0,0 +1,40 @@
|
||||
# Copyright 2020 Canonical Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import mock
|
||||
import unittest
|
||||
import sys
|
||||
|
||||
import zaza.openstack.charm_tests.rabbitmq_server.utils as rabbit_utils
|
||||
|
||||
|
||||
class TestRabbitUtils(unittest.TestCase):
|
||||
"""Test class to encapsulate testing Mysql test utils."""
|
||||
|
||||
def setUp(self):
|
||||
super(TestRabbitUtils, self).setUp()
|
||||
if sys.version_info < (3, 6, 0):
|
||||
raise unittest.SkipTest("Can't AsyncMock in py35")
|
||||
|
||||
@mock.patch.object(rabbit_utils.zaza, 'model')
|
||||
def test_rabbit_complete_cluster_series_upgrade(self, mock_model):
|
||||
run_action_on_leader = mock.AsyncMock()
|
||||
mock_model.async_run_action_on_leader = run_action_on_leader
|
||||
asyncio.get_event_loop().run_until_complete(
|
||||
rabbit_utils.complete_cluster_series_upgrade())
|
||||
run_action_on_leader.assert_called_once_with(
|
||||
'rabbitmq-server',
|
||||
'complete-cluster-series-upgrade',
|
||||
action_params={})
|
||||
@@ -0,0 +1,559 @@
|
||||
# Copyright 2020 Canonical Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import mock
|
||||
import sys
|
||||
import unittest
|
||||
import unit_tests.utils as ut_utils
|
||||
import zaza.openstack.utilities.generic as generic_utils
|
||||
import zaza.openstack.utilities.series_upgrade as series_upgrade
|
||||
import zaza.openstack.utilities.parallel_series_upgrade as upgrade_utils
|
||||
|
||||
FAKE_STATUS = {
|
||||
'can-upgrade-to': '',
|
||||
'charm': 'local:trusty/app-136',
|
||||
'subordinate-to': [],
|
||||
'units': {'app/0': {'leader': True,
|
||||
'machine': '0',
|
||||
'subordinates': {
|
||||
'app-hacluster/0': {
|
||||
'charm': 'local:trusty/hacluster-0',
|
||||
'leader': True}}},
|
||||
'app/1': {'machine': '1',
|
||||
'subordinates': {
|
||||
'app-hacluster/1': {
|
||||
'charm': 'local:trusty/hacluster-0'}}},
|
||||
'app/2': {'machine': '2',
|
||||
'subordinates': {
|
||||
'app-hacluster/2': {
|
||||
'charm': 'local:trusty/hacluster-0'}}}}}
|
||||
|
||||
FAKE_STATUS_MONGO = {
|
||||
'can-upgrade-to': '',
|
||||
'charm': 'local:trusty/mongodb-10',
|
||||
'subordinate-to': [],
|
||||
'units': {'mongo/0': {'leader': True,
|
||||
'machine': '0',
|
||||
'subordinates': {}},
|
||||
'mongo/1': {'machine': '1',
|
||||
'subordinates': {}},
|
||||
'mongo/2': {'machine': '2',
|
||||
'subordinates': {}}}}
|
||||
|
||||
|
||||
class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(Test_ParallelSeriesUpgradeSync, self).setUp()
|
||||
# Juju Status Object and data
|
||||
# self.juju_status = mock.MagicMock()
|
||||
# self.juju_status.applications.__getitem__.return_value = FAKE_STATUS
|
||||
# self.patch_object(upgrade_utils, "model")
|
||||
# self.model.get_status.return_value = self.juju_status
|
||||
|
||||
def test_get_leader_and_non_leaders(self):
|
||||
expected = ({
|
||||
'app/0': {
|
||||
'leader': True,
|
||||
'machine': '0',
|
||||
'subordinates': {
|
||||
'app-hacluster/0': {
|
||||
'charm': 'local:trusty/hacluster-0',
|
||||
'leader': True}}}}, {
|
||||
'app/1': {
|
||||
'machine': '1',
|
||||
'subordinates': {
|
||||
'app-hacluster/1': {
|
||||
'charm': 'local:trusty/hacluster-0'}}},
|
||||
'app/2': {
|
||||
'machine': '2',
|
||||
'subordinates': {
|
||||
'app-hacluster/2': {
|
||||
'charm': 'local:trusty/hacluster-0'}}}})
|
||||
|
||||
self.assertEqual(
|
||||
expected,
|
||||
upgrade_utils.get_leader_and_non_leaders(FAKE_STATUS)
|
||||
)
|
||||
|
||||
def test_app_config_openstack_charm(self):
|
||||
expected = {
|
||||
'origin': 'openstack-origin',
|
||||
'pause_non_leader_subordinate': True,
|
||||
'pause_non_leader_primary': True,
|
||||
'post_upgrade_functions': [],
|
||||
'pre_upgrade_functions': [],
|
||||
'post_application_upgrade_functions': [],
|
||||
'follower_first': False, }
|
||||
config = upgrade_utils.app_config('keystone')
|
||||
self.assertEqual(expected, config)
|
||||
|
||||
def test_app_config_mongo(self):
|
||||
expected = {
|
||||
'origin': None,
|
||||
'pause_non_leader_subordinate': True,
|
||||
'pause_non_leader_primary': True,
|
||||
'post_upgrade_functions': [],
|
||||
'pre_upgrade_functions': [],
|
||||
'post_application_upgrade_functions': [],
|
||||
'follower_first': True, }
|
||||
config = upgrade_utils.app_config('mongodb')
|
||||
self.assertEqual(expected, config)
|
||||
|
||||
def test_app_config_ceph(self):
|
||||
expected = {
|
||||
'origin': 'source',
|
||||
'pause_non_leader_subordinate': False,
|
||||
'pause_non_leader_primary': False,
|
||||
'post_upgrade_functions': [],
|
||||
'pre_upgrade_functions': [],
|
||||
'post_application_upgrade_functions': [],
|
||||
'follower_first': False, }
|
||||
config = upgrade_utils.app_config('ceph-mon')
|
||||
self.assertEqual(expected, config)
|
||||
|
||||
def test_app_config_percona(self):
|
||||
expected = {
|
||||
'origin': 'source',
|
||||
'pause_non_leader_subordinate': True,
|
||||
'pause_non_leader_primary': True,
|
||||
'post_upgrade_functions': [],
|
||||
'pre_upgrade_functions': [],
|
||||
'post_application_upgrade_functions': [
|
||||
('zaza.openstack.charm_tests.mysql.utils.'
|
||||
'complete_cluster_series_upgrade')
|
||||
],
|
||||
'follower_first': False, }
|
||||
config = upgrade_utils.app_config('percona-cluster')
|
||||
self.assertEqual(expected, config)
|
||||
|
||||
|
||||
class AioTestCase(ut_utils.BaseTestCase):
|
||||
def __init__(self, methodName='runTest', loop=None):
|
||||
self.loop = loop or asyncio.get_event_loop()
|
||||
self._function_cache = {}
|
||||
super(AioTestCase, self).__init__(methodName=methodName)
|
||||
|
||||
def coroutine_function_decorator(self, func):
|
||||
def wrapper(*args, **kw):
|
||||
return self.loop.run_until_complete(func(*args, **kw))
|
||||
return wrapper
|
||||
|
||||
def __getattribute__(self, item):
|
||||
attr = object.__getattribute__(self, item)
|
||||
if asyncio.iscoroutinefunction(attr) and item.startswith('test_'):
|
||||
if item not in self._function_cache:
|
||||
self._function_cache[item] = (
|
||||
self.coroutine_function_decorator(attr))
|
||||
return self._function_cache[item]
|
||||
return attr
|
||||
|
||||
|
||||
class TestParallelSeriesUpgrade(AioTestCase):
|
||||
def setUp(self):
|
||||
super(TestParallelSeriesUpgrade, self).setUp()
|
||||
if sys.version_info < (3, 6, 0):
|
||||
raise unittest.SkipTest("Can't AsyncMock in py35")
|
||||
self.patch_object(series_upgrade, "async_prepare_series_upgrade")
|
||||
self.patch_object(generic_utils, 'check_call')
|
||||
# Juju Status Object and data
|
||||
|
||||
self.juju_status = mock.AsyncMock()
|
||||
self.juju_status.return_value.applications.__getitem__.return_value = \
|
||||
FAKE_STATUS
|
||||
self.patch_object(upgrade_utils, "model")
|
||||
self.model.async_get_status = self.juju_status
|
||||
self.async_run_action = mock.AsyncMock()
|
||||
self.model.async_run_action = self.async_run_action
|
||||
|
||||
self.async_block_until = mock.AsyncMock()
|
||||
self.model.async_block_until = self.async_block_until
|
||||
self.model.async_wait_for_unit_idle = mock.AsyncMock()
|
||||
self.async_run_on_machine = mock.AsyncMock()
|
||||
self.model.async_run_on_machine = self.async_run_on_machine
|
||||
|
||||
@mock.patch.object(upgrade_utils.cl_utils, 'get_class')
|
||||
async def test_run_post_application_upgrade_functions(
|
||||
self,
|
||||
mock_get_class
|
||||
):
|
||||
called = mock.AsyncMock()
|
||||
mock_get_class.return_value = called
|
||||
await upgrade_utils.run_post_application_upgrade_functions(
|
||||
['my.thing'])
|
||||
mock_get_class.assert_called_once_with('my.thing')
|
||||
called.assert_called()
|
||||
|
||||
@mock.patch.object(upgrade_utils.cl_utils, 'get_class')
|
||||
async def test_run_pre_upgrade_functions(self, mock_get_class):
|
||||
called = mock.AsyncMock()
|
||||
mock_get_class.return_value = called
|
||||
await upgrade_utils.run_pre_upgrade_functions('1', ['my.thing'])
|
||||
mock_get_class.assert_called_once_with('my.thing')
|
||||
called.assert_called_once_with('1')
|
||||
|
||||
@mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions')
|
||||
@mock.patch.object(
|
||||
upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade')
|
||||
@mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series')
|
||||
@mock.patch.object(upgrade_utils, 'maybe_pause_things')
|
||||
@mock.patch.object(upgrade_utils, 'series_upgrade_machine')
|
||||
async def test_parallel_series_upgrade_mongo(
|
||||
self,
|
||||
mock_series_upgrade_machine,
|
||||
mock_maybe_pause_things,
|
||||
mock_async_set_series,
|
||||
mock_async_prepare_series_upgrade,
|
||||
mock_post_application_upgrade_functions,
|
||||
):
|
||||
self.juju_status.return_value.applications.__getitem__.return_value = \
|
||||
FAKE_STATUS_MONGO
|
||||
upgrade_config = upgrade_utils.app_config('mongodb')
|
||||
await upgrade_utils.parallel_series_upgrade(
|
||||
'mongodb',
|
||||
from_series='trusty',
|
||||
to_series='xenial',
|
||||
**upgrade_config
|
||||
)
|
||||
mock_async_set_series.assert_called_once_with(
|
||||
'mongodb', to_series='xenial')
|
||||
self.juju_status.assert_called()
|
||||
|
||||
# The below is using `any_order=True` because the ordering is
|
||||
# undetermined and differs between python versions
|
||||
mock_async_prepare_series_upgrade.assert_has_calls([
|
||||
mock.call('1', to_series='xenial'),
|
||||
mock.call('2', to_series='xenial'),
|
||||
mock.call('0', to_series='xenial'),
|
||||
], any_order=True)
|
||||
mock_maybe_pause_things.assert_called()
|
||||
mock_series_upgrade_machine.assert_has_calls([
|
||||
mock.call(
|
||||
'1',
|
||||
origin=None,
|
||||
application='mongodb',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=[]),
|
||||
mock.call(
|
||||
'2',
|
||||
origin=None,
|
||||
application='mongodb',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=[]),
|
||||
mock.call(
|
||||
'0',
|
||||
origin=None,
|
||||
application='mongodb',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=[]),
|
||||
])
|
||||
mock_post_application_upgrade_functions.assert_called_once_with([])
|
||||
|
||||
@mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions')
|
||||
@mock.patch.object(
|
||||
upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade')
|
||||
@mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series')
|
||||
@mock.patch.object(upgrade_utils, 'maybe_pause_things')
|
||||
@mock.patch.object(upgrade_utils, 'series_upgrade_machine')
|
||||
async def test_serial_series_upgrade_mongo(
|
||||
self,
|
||||
mock_series_upgrade_machine,
|
||||
mock_maybe_pause_things,
|
||||
mock_async_set_series,
|
||||
mock_async_prepare_series_upgrade,
|
||||
mock_post_application_upgrade_functions,
|
||||
):
|
||||
self.juju_status.return_value.applications.__getitem__.return_value = \
|
||||
FAKE_STATUS_MONGO
|
||||
upgrade_config = upgrade_utils.app_config('mongodb')
|
||||
await upgrade_utils.serial_series_upgrade(
|
||||
'mongodb',
|
||||
from_series='trusty',
|
||||
to_series='xenial',
|
||||
**upgrade_config
|
||||
)
|
||||
mock_async_set_series.assert_called_once_with(
|
||||
'mongodb', to_series='xenial')
|
||||
self.juju_status.assert_called()
|
||||
mock_async_prepare_series_upgrade.assert_has_calls([
|
||||
mock.call('1', to_series='xenial'),
|
||||
mock.call('2', to_series='xenial'),
|
||||
mock.call('0', to_series='xenial'),
|
||||
])
|
||||
mock_maybe_pause_things.assert_called()
|
||||
mock_series_upgrade_machine.assert_has_calls([
|
||||
mock.call(
|
||||
'1',
|
||||
origin=None,
|
||||
application='mongodb',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=[]),
|
||||
mock.call(
|
||||
'2',
|
||||
origin=None,
|
||||
application='mongodb',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=[]),
|
||||
mock.call(
|
||||
'0',
|
||||
origin=None,
|
||||
application='mongodb',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=[]),
|
||||
])
|
||||
mock_post_application_upgrade_functions.assert_called_once_with([])
|
||||
|
||||
@mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions')
|
||||
@mock.patch.object(
|
||||
upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade')
|
||||
@mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series')
|
||||
@mock.patch.object(upgrade_utils, 'maybe_pause_things')
|
||||
@mock.patch.object(upgrade_utils, 'series_upgrade_machine')
|
||||
async def test_parallel_series_upgrade(
|
||||
self,
|
||||
mock_series_upgrade_machine,
|
||||
mock_maybe_pause_things,
|
||||
mock_async_set_series,
|
||||
mock_async_prepare_series_upgrade,
|
||||
mock_post_application_upgrade_functions,
|
||||
):
|
||||
await upgrade_utils.parallel_series_upgrade(
|
||||
'app',
|
||||
from_series='trusty',
|
||||
to_series='xenial',
|
||||
)
|
||||
mock_async_set_series.assert_called_once_with(
|
||||
'app', to_series='xenial')
|
||||
self.juju_status.assert_called()
|
||||
# The below is using `any_order=True` because the ordering is
|
||||
# undetermined and differs between python versions
|
||||
mock_async_prepare_series_upgrade.assert_has_calls([
|
||||
mock.call('1', to_series='xenial'),
|
||||
mock.call('2', to_series='xenial'),
|
||||
mock.call('0', to_series='xenial'),
|
||||
], any_order=True)
|
||||
mock_maybe_pause_things.assert_called()
|
||||
mock_series_upgrade_machine.assert_has_calls([
|
||||
mock.call(
|
||||
'1',
|
||||
origin='openstack-origin',
|
||||
application='app',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=None),
|
||||
mock.call(
|
||||
'2',
|
||||
origin='openstack-origin',
|
||||
application='app',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=None),
|
||||
mock.call(
|
||||
'0',
|
||||
origin='openstack-origin',
|
||||
application='app',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=None),
|
||||
])
|
||||
mock_post_application_upgrade_functions.assert_called_once_with(None)
|
||||
|
||||
@mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions')
|
||||
@mock.patch.object(
|
||||
upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade')
|
||||
@mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series')
|
||||
@mock.patch.object(upgrade_utils, 'maybe_pause_things')
|
||||
@mock.patch.object(upgrade_utils, 'series_upgrade_machine')
|
||||
async def test_serial_series_upgrade(
|
||||
self,
|
||||
mock_series_upgrade_machine,
|
||||
mock_maybe_pause_things,
|
||||
mock_async_set_series,
|
||||
mock_async_prepare_series_upgrade,
|
||||
mock_post_application_upgrade_functions,
|
||||
):
|
||||
await upgrade_utils.serial_series_upgrade(
|
||||
'app',
|
||||
from_series='trusty',
|
||||
to_series='xenial',
|
||||
)
|
||||
mock_async_set_series.assert_called_once_with(
|
||||
'app', to_series='xenial')
|
||||
self.juju_status.assert_called()
|
||||
mock_async_prepare_series_upgrade.assert_has_calls([
|
||||
mock.call('0', to_series='xenial'),
|
||||
mock.call('1', to_series='xenial'),
|
||||
mock.call('2', to_series='xenial'),
|
||||
])
|
||||
mock_maybe_pause_things.assert_called()
|
||||
mock_series_upgrade_machine.assert_has_calls([
|
||||
mock.call(
|
||||
'0',
|
||||
origin='openstack-origin',
|
||||
application='app',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=None),
|
||||
mock.call(
|
||||
'1',
|
||||
origin='openstack-origin',
|
||||
application='app',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=None),
|
||||
mock.call(
|
||||
'2',
|
||||
origin='openstack-origin',
|
||||
application='app',
|
||||
files=None,
|
||||
workaround_script=None,
|
||||
post_upgrade_functions=None),
|
||||
])
|
||||
mock_post_application_upgrade_functions.assert_called_once_with(None)
|
||||
|
||||
@mock.patch.object(
|
||||
upgrade_utils.series_upgrade_utils, 'async_complete_series_upgrade')
|
||||
@mock.patch.object(upgrade_utils, 'reboot')
|
||||
@mock.patch.object(upgrade_utils, 'async_do_release_upgrade')
|
||||
@mock.patch.object(upgrade_utils, 'async_dist_upgrade')
|
||||
async def test_series_upgrade_machine(
|
||||
self,
|
||||
mock_async_dist_upgrade,
|
||||
mock_async_do_release_upgrade,
|
||||
mock_reboot,
|
||||
mock_async_complete_series_upgrade
|
||||
):
|
||||
await upgrade_utils.series_upgrade_machine(
|
||||
'1',
|
||||
post_upgrade_functions=None,
|
||||
pre_upgrade_functions=None,
|
||||
files=None,
|
||||
workaround_script=None)
|
||||
mock_async_dist_upgrade.assert_called_once_with('1')
|
||||
mock_async_do_release_upgrade.assert_called_once_with('1')
|
||||
mock_reboot.assert_called_once_with('1')
|
||||
mock_async_complete_series_upgrade.assert_called_once_with('1')
|
||||
|
||||
@mock.patch.object(upgrade_utils.os_utils, 'async_set_origin')
|
||||
@mock.patch.object(
|
||||
upgrade_utils.series_upgrade_utils, 'async_complete_series_upgrade')
|
||||
@mock.patch.object(upgrade_utils, 'reboot')
|
||||
@mock.patch.object(upgrade_utils, 'async_do_release_upgrade')
|
||||
@mock.patch.object(upgrade_utils, 'async_dist_upgrade')
|
||||
async def test_series_upgrade_machine_with_source(
|
||||
self,
|
||||
mock_async_dist_upgrade,
|
||||
mock_async_do_release_upgrade,
|
||||
mock_reboot,
|
||||
mock_async_complete_series_upgrade,
|
||||
mock_async_set_origin
|
||||
):
|
||||
await upgrade_utils.series_upgrade_machine(
|
||||
'1',
|
||||
origin='openstack-origin',
|
||||
application='app',
|
||||
post_upgrade_functions=None,
|
||||
pre_upgrade_functions=None,
|
||||
files=None,
|
||||
workaround_script=None)
|
||||
mock_async_dist_upgrade.assert_called_once_with('1')
|
||||
mock_async_do_release_upgrade.assert_called_once_with('1')
|
||||
mock_reboot.assert_called_once_with('1')
|
||||
mock_async_complete_series_upgrade.assert_called_once_with('1')
|
||||
mock_async_set_origin.assert_called_once_with(
|
||||
'app', 'openstack-origin')
|
||||
|
||||
async def test_maybe_pause_things_primary(self):
|
||||
await upgrade_utils.maybe_pause_things(
|
||||
FAKE_STATUS,
|
||||
['app/1', 'app/2'],
|
||||
pause_non_leader_subordinate=False,
|
||||
pause_non_leader_primary=True)
|
||||
self.async_run_action.assert_has_calls([
|
||||
mock.call('app/1', "pause", action_params={}),
|
||||
mock.call('app/2', "pause", action_params={}),
|
||||
])
|
||||
|
||||
async def test_maybe_pause_things_subordinates(self):
|
||||
await upgrade_utils.maybe_pause_things(
|
||||
FAKE_STATUS,
|
||||
['app/1', 'app/2'],
|
||||
pause_non_leader_subordinate=True,
|
||||
pause_non_leader_primary=False)
|
||||
self.async_run_action.assert_has_calls([
|
||||
mock.call('app-hacluster/1', "pause", action_params={}),
|
||||
mock.call('app-hacluster/2', "pause", action_params={}),
|
||||
])
|
||||
|
||||
async def test_maybe_pause_things_all(self):
|
||||
await upgrade_utils.maybe_pause_things(
|
||||
FAKE_STATUS,
|
||||
['app/1', 'app/2'],
|
||||
pause_non_leader_subordinate=True,
|
||||
pause_non_leader_primary=True)
|
||||
self.async_run_action.assert_has_calls([
|
||||
mock.call('app-hacluster/1', "pause", action_params={}),
|
||||
mock.call('app/1', "pause", action_params={}),
|
||||
mock.call('app-hacluster/2', "pause", action_params={}),
|
||||
mock.call('app/2', "pause", action_params={}),
|
||||
])
|
||||
|
||||
async def test_maybe_pause_things_none(self):
|
||||
await upgrade_utils.maybe_pause_things(
|
||||
FAKE_STATUS,
|
||||
['app/1', 'app/2'],
|
||||
pause_non_leader_subordinate=False,
|
||||
pause_non_leader_primary=False)
|
||||
self.async_run_action.assert_not_called()
|
||||
|
||||
async def test_async_do_release_upgrade(self):
|
||||
await upgrade_utils.async_do_release_upgrade('1')
|
||||
do_release_upgrade_cmd = (
|
||||
'yes | sudo DEBIAN_FRONTEND=noninteractive '
|
||||
'do-release-upgrade -f DistUpgradeViewNonInteractive')
|
||||
self.async_run_on_machine.assert_called_once_with(
|
||||
'1', do_release_upgrade_cmd, timeout='120m'
|
||||
)
|
||||
|
||||
async def test_prepare_series_upgrade(self):
|
||||
await upgrade_utils.prepare_series_upgrade(
|
||||
'1', to_series='xenial'
|
||||
)
|
||||
self.async_prepare_series_upgrade.assert_called_once_with(
|
||||
'1', to_series='xenial'
|
||||
)
|
||||
|
||||
async def test_reboot(self):
|
||||
await upgrade_utils.reboot('1')
|
||||
self.async_run_on_machine.assert_called_once_with(
|
||||
'1', 'sudo init 6 & exit'
|
||||
)
|
||||
|
||||
async def test_async_dist_upgrade(self):
|
||||
await upgrade_utils.async_dist_upgrade('1')
|
||||
apt_update_command = (
|
||||
"""yes | sudo DEBIAN_FRONTEND=noninteractive """
|
||||
"""apt-get --assume-yes """
|
||||
"""-o "Dpkg::Options::=--force-confdef" """
|
||||
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
|
||||
self.async_run_on_machine.assert_has_calls([
|
||||
mock.call('1', 'sudo apt-get update'),
|
||||
mock.call('1', apt_update_command),
|
||||
])
|
||||
@@ -90,6 +90,7 @@ class TestUpgradeUtils(ut_utils.BaseTestCase):
|
||||
|
||||
def test_get_upgrade_groups(self):
|
||||
expected = collections.OrderedDict([
|
||||
('Stateful Services', []),
|
||||
('Core Identity', []),
|
||||
('Control Plane', ['cinder']),
|
||||
('Data Plane', ['nova-compute']),
|
||||
@@ -103,10 +104,11 @@ class TestUpgradeUtils(ut_utils.BaseTestCase):
|
||||
|
||||
def test_get_series_upgrade_groups(self):
|
||||
expected = collections.OrderedDict([
|
||||
('Stateful Services', ['mydb']),
|
||||
('Core Identity', []),
|
||||
('Control Plane', ['cinder']),
|
||||
('Data Plane', ['nova-compute']),
|
||||
('sweep_up', ['mydb', 'ntp'])])
|
||||
('sweep_up', ['ntp'])])
|
||||
actual = openstack_upgrade.get_series_upgrade_groups()
|
||||
pprint.pprint(expected)
|
||||
pprint.pprint(actual)
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# Copyright 2020 Canonical Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Module of functions for interfacing with the percona-cluster charm."""
|
||||
|
||||
import zaza.model as model
|
||||
|
||||
|
||||
async def complete_cluster_series_upgrade():
|
||||
"""Run the complete-cluster-series-upgrade action on the lead unit."""
|
||||
# TODO: Make this work across either mysql or percona-cluster names
|
||||
await model.async_run_action_on_leader(
|
||||
'mysql',
|
||||
'complete-cluster-series-upgrade',
|
||||
action_params={})
|
||||
@@ -584,3 +584,11 @@ def _post_check_unit_cluster_nodes(unit, nodes, unit_node_names):
|
||||
'after unit removal.\n'
|
||||
''.format(unit_name, node))
|
||||
return errors
|
||||
|
||||
|
||||
async def complete_cluster_series_upgrade():
|
||||
"""Run the complete-cluster-series-upgrade action on the lead unit."""
|
||||
await zaza.model.async_run_action_on_leader(
|
||||
'rabbitmq-server',
|
||||
'complete-cluster-series-upgrade',
|
||||
action_params={})
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright 2018 Canonical Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Define class for Series Upgrade."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
from zaza import model
|
||||
from zaza.openstack.utilities import (
|
||||
cli as cli_utils,
|
||||
upgrade_utils as upgrade_utils,
|
||||
)
|
||||
from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest
|
||||
from zaza.openstack.utilities import (
|
||||
parallel_series_upgrade,
|
||||
)
|
||||
|
||||
|
||||
def _filter_easyrsa(app, app_config, model_name=None):
|
||||
charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm'])
|
||||
if "easyrsa" in charm_name:
|
||||
logging.warn("Skipping series upgrade of easyrsa Bug #1850121")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _filter_etcd(app, app_config, model_name=None):
|
||||
charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm'])
|
||||
if "etcd" in charm_name:
|
||||
logging.warn("Skipping series upgrade of easyrsa Bug #1850124")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class ParallelSeriesUpgradeTest(unittest.TestCase):
|
||||
"""Class to encapsulate Series Upgrade Tests."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for Series Upgrades."""
|
||||
cli_utils.setup_logging()
|
||||
cls.from_series = None
|
||||
cls.to_series = None
|
||||
cls.workaround_script = None
|
||||
cls.files = []
|
||||
|
||||
def test_200_run_series_upgrade(self):
|
||||
"""Run series upgrade."""
|
||||
# Set Feature Flag
|
||||
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
|
||||
upgrade_groups = upgrade_utils.get_series_upgrade_groups(
|
||||
extra_filters=[_filter_etcd, _filter_easyrsa])
|
||||
from_series = self.from_series
|
||||
to_series = self.to_series
|
||||
completed_machines = []
|
||||
workaround_script = None
|
||||
files = []
|
||||
applications = model.get_status().applications
|
||||
for group_name, apps in upgrade_groups.items():
|
||||
logging.info("About to upgrade {} from {} to {}".format(
|
||||
group_name, from_series, to_series))
|
||||
upgrade_functions = []
|
||||
if group_name in ["Stateful Services", "Data Plane", "sweep_up"]:
|
||||
logging.info("Going to upgrade {} unit by unit".format(apps))
|
||||
upgrade_function = \
|
||||
parallel_series_upgrade.serial_series_upgrade
|
||||
else:
|
||||
logging.info("Going to upgrade {} all at once".format(apps))
|
||||
upgrade_function = \
|
||||
parallel_series_upgrade.parallel_series_upgrade
|
||||
|
||||
for charm_name in apps:
|
||||
charm = applications[charm_name]['charm']
|
||||
name = upgrade_utils.extract_charm_name_from_url(charm)
|
||||
upgrade_config = parallel_series_upgrade.app_config(name)
|
||||
upgrade_functions.append(
|
||||
upgrade_function(
|
||||
charm_name,
|
||||
**upgrade_config,
|
||||
from_series=from_series,
|
||||
to_series=to_series,
|
||||
completed_machines=completed_machines,
|
||||
workaround_script=workaround_script,
|
||||
files=files))
|
||||
asyncio.get_event_loop().run_until_complete(
|
||||
asyncio.gather(*upgrade_functions))
|
||||
model.block_until_all_units_idle()
|
||||
logging.info("Finished {}".format(group_name))
|
||||
logging.info("Done!")
|
||||
|
||||
|
||||
class OpenStackParallelSeriesUpgrade(ParallelSeriesUpgradeTest):
|
||||
"""OpenStack Series Upgrade.
|
||||
|
||||
Full OpenStack series upgrade with VM launch before and after the series
|
||||
upgrade.
|
||||
|
||||
This test requires a full OpenStack including at least: keystone, glance,
|
||||
nova-cloud-controller, nova-compute, neutron-gateway, neutron-api and
|
||||
neutron-openvswitch.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for Series Upgrades."""
|
||||
super(OpenStackParallelSeriesUpgrade, cls).setUpClass()
|
||||
cls.lts = LTSGuestCreateTest()
|
||||
cls.lts.setUpClass()
|
||||
|
||||
def test_100_validate_pre_series_upgrade_cloud(self):
|
||||
"""Validate pre series upgrade."""
|
||||
logging.info("Validate pre-series-upgrade: Spin up LTS instance")
|
||||
self.lts.test_launch_small_instance()
|
||||
|
||||
def test_500_validate_series_upgraded_cloud(self):
|
||||
"""Validate post series upgrade."""
|
||||
logging.info("Validate post-series-upgrade: Spin up LTS instance")
|
||||
self.lts.test_launch_small_instance()
|
||||
|
||||
|
||||
class TrustyXenialSeriesUpgrade(OpenStackParallelSeriesUpgrade):
|
||||
"""OpenStack Trusty to Xenial Series Upgrade."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for Trusty to Xenial Series Upgrades."""
|
||||
super(TrustyXenialSeriesUpgrade, cls).setUpClass()
|
||||
cls.from_series = "trusty"
|
||||
cls.to_series = "xenial"
|
||||
|
||||
|
||||
class XenialBionicSeriesUpgrade(OpenStackParallelSeriesUpgrade):
|
||||
"""OpenStack Xenial to Bionic Series Upgrade."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for Xenial to Bionic Series Upgrades."""
|
||||
super(XenialBionicSeriesUpgrade, cls).setUpClass()
|
||||
cls.from_series = "xenial"
|
||||
cls.to_series = "bionic"
|
||||
|
||||
|
||||
class BionicFocalSeriesUpgrade(OpenStackParallelSeriesUpgrade):
|
||||
"""OpenStack Bionic to FocalSeries Upgrade."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for Xenial to Bionic Series Upgrades."""
|
||||
super(BionicFocalSeriesUpgrade, cls).setUpClass()
|
||||
cls.from_series = "bionic"
|
||||
cls.to_series = "focal"
|
||||
|
||||
|
||||
class UbuntuLiteParallelSeriesUpgrade(unittest.TestCase):
|
||||
"""ubuntu Lite Parallel Series Upgrade."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for Series Upgrades."""
|
||||
cli_utils.setup_logging()
|
||||
cls.from_series = None
|
||||
cls.to_series = None
|
||||
|
||||
def test_200_run_series_upgrade(self):
|
||||
"""Run series upgrade."""
|
||||
# Set Feature Flag
|
||||
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
|
||||
parallel_series_upgrade.upgrade_ubuntu_lite(
|
||||
from_series=self.from_series,
|
||||
to_series=self.to_series
|
||||
)
|
||||
|
||||
|
||||
class TrustyXenialSeriesUpgradeUbuntu(UbuntuLiteParallelSeriesUpgrade):
|
||||
"""OpenStack Trusty to Xenial Series Upgrade."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for Trusty to Xenial Series Upgrades."""
|
||||
super(TrustyXenialSeriesUpgradeUbuntu, cls).setUpClass()
|
||||
cls.from_series = "trusty"
|
||||
cls.to_series = "xenial"
|
||||
|
||||
|
||||
class XenialBionicSeriesUpgradeUbuntu(UbuntuLiteParallelSeriesUpgrade):
|
||||
"""OpenStack Xenial to Bionic Series Upgrade."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for Xenial to Bionic Series Upgrades."""
|
||||
super(XenialBionicSeriesUpgradeUbuntu, cls).setUpClass()
|
||||
cls.from_series = "xenial"
|
||||
cls.to_series = "bionic"
|
||||
|
||||
|
||||
class BionicFocalSeriesUpgradeUbuntu(UbuntuLiteParallelSeriesUpgrade):
|
||||
"""OpenStack Bionic to FocalSeries Upgrade."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run setup for Xenial to Bionic Series Upgrades."""
|
||||
super(BionicFocalSeriesUpgradeUbuntu, cls).setUpClass()
|
||||
cls.from_series = "bionic"
|
||||
cls.to_series = "focal"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from_series = os.environ.get("FROM_SERIES")
|
||||
if from_series == "trusty":
|
||||
to_series = "xenial"
|
||||
series_upgrade_test = TrustyXenialSeriesUpgrade()
|
||||
elif from_series == "xenial":
|
||||
to_series = "bionic"
|
||||
series_upgrade_test = XenialBionicSeriesUpgrade()
|
||||
elif from_series == "bionic":
|
||||
to_series = "focal"
|
||||
series_upgrade_test = BionicFocalSeriesUpgrade()
|
||||
|
||||
else:
|
||||
raise Exception("FROM_SERIES is not set to a vailid LTS series")
|
||||
series_upgrade_test.setUpClass()
|
||||
sys.exit(series_upgrade_test.test_200_run_series_upgrade())
|
||||
@@ -66,6 +66,20 @@ def mojo_unseal_by_unit():
|
||||
zaza.model.run_on_unit(unit_name, './hooks/update-status')
|
||||
|
||||
|
||||
async def async_mojo_unseal_by_unit():
|
||||
"""Unseal any units reported as sealed using mojo cacert."""
|
||||
cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path()
|
||||
vault_creds = vault_utils.get_credentails()
|
||||
for client in vault_utils.get_clients(cacert=cacert):
|
||||
if client.hvac_client.is_sealed():
|
||||
client.hvac_client.unseal(vault_creds['keys'][0])
|
||||
unit_name = await juju_utils.async_get_unit_name_from_ip_address(
|
||||
client.addr,
|
||||
'vault')
|
||||
await zaza.model.async_run_on_unit(
|
||||
unit_name, './hooks/update-status')
|
||||
|
||||
|
||||
def auto_initialize(cacert=None, validation_application='keystone'):
|
||||
"""Auto initialize vault for testing.
|
||||
|
||||
|
||||
@@ -385,10 +385,17 @@ async def check_call(cmd):
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE)
|
||||
stdout, stderr = await proc.communicate()
|
||||
stdout = stdout.decode('utf-8')
|
||||
stderr = stderr.decode('utf-8')
|
||||
if proc.returncode != 0:
|
||||
logging.warn("STDOUT: {}".format(stdout))
|
||||
logging.warn("STDERR: {}".format(stderr))
|
||||
raise subprocess.CalledProcessError(proc.returncode, cmd)
|
||||
else:
|
||||
if stderr:
|
||||
logging.info("STDERR: {} ({})".format(stderr, ' '.join(cmd)))
|
||||
if stdout:
|
||||
logging.info("STDOUT: {} ({})".format(stdout, ' '.join(cmd)))
|
||||
|
||||
|
||||
def set_dpkg_non_interactive_on_unit(
|
||||
|
||||
+573
@@ -0,0 +1,573 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2020 Canonical Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Collection of functions for testing series upgrade in parallel."""
|
||||
|
||||
|
||||
import asyncio
|
||||
|
||||
import collections
|
||||
import copy
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
from zaza import model
|
||||
from zaza.charm_lifecycle import utils as cl_utils
|
||||
import zaza.openstack.utilities.generic as os_utils
|
||||
import zaza.openstack.utilities.series_upgrade as series_upgrade_utils
|
||||
from zaza.openstack.utilities.series_upgrade import (
|
||||
SUBORDINATE_PAUSE_RESUME_BLACKLIST,
|
||||
)
|
||||
|
||||
|
||||
def app_config(charm_name):
|
||||
"""Return a dict with the upgrade config for an application.
|
||||
|
||||
:param charm_name: Name of the charm about to upgrade
|
||||
:type charm_name: str
|
||||
:param async: Whether the upgreade functions should be async
|
||||
:type async: bool
|
||||
:returns: A dicitonary of the upgrade config for the application
|
||||
:rtype: Dict
|
||||
"""
|
||||
default = {
|
||||
'origin': 'openstack-origin',
|
||||
'pause_non_leader_subordinate': True,
|
||||
'pause_non_leader_primary': True,
|
||||
'post_upgrade_functions': [],
|
||||
'pre_upgrade_functions': [],
|
||||
'post_application_upgrade_functions': [],
|
||||
'follower_first': False, }
|
||||
_app_settings = collections.defaultdict(lambda: default)
|
||||
ceph = {
|
||||
'origin': "source",
|
||||
'pause_non_leader_primary': False,
|
||||
'pause_non_leader_subordinate': False,
|
||||
}
|
||||
exceptions = {
|
||||
'rabbitmq-server': {
|
||||
'origin': 'source',
|
||||
'pause_non_leader_subordinate': False,
|
||||
'post_application_upgrade_functions': [
|
||||
('zaza.openstack.charm_tests.rabbitmq_server.utils.'
|
||||
'complete_cluster_series_upgrade')]
|
||||
},
|
||||
'percona-cluster': {
|
||||
'origin': 'source',
|
||||
'post_application_upgrade_functions': [
|
||||
('zaza.openstack.charm_tests.mysql.utils.'
|
||||
'complete_cluster_series_upgrade')]
|
||||
},
|
||||
'nova-compute': {
|
||||
'pause_non_leader_primary': False,
|
||||
'pause_non_leader_subordinate': False,
|
||||
# TODO
|
||||
# 'pre_upgrade_functions': [
|
||||
# 'zaza.openstack.charm_tests.nova_compute.setup.evacuate'
|
||||
# ]
|
||||
},
|
||||
'ceph': ceph,
|
||||
'ceph-mon': ceph,
|
||||
'ceph-osd': ceph,
|
||||
'designate-bind': {'origin': None, },
|
||||
'tempest': {'origin': None, },
|
||||
'memcached': {
|
||||
'origin': None,
|
||||
'pause_non_leader_primary': False,
|
||||
'pause_non_leader_subordinate': False,
|
||||
},
|
||||
'vault': {
|
||||
'origin': None,
|
||||
'pause_non_leader_primary': False,
|
||||
'pause_non_leader_subordinate': True,
|
||||
'post_upgrade_functions': [
|
||||
('zaza.openstack.charm_tests.vault.setup.'
|
||||
'async_mojo_unseal_by_unit')]
|
||||
},
|
||||
'mongodb': {
|
||||
'origin': None,
|
||||
'follower_first': True,
|
||||
}
|
||||
}
|
||||
for key, value in exceptions.items():
|
||||
_app_settings[key] = copy.deepcopy(default)
|
||||
_app_settings[key].update(value)
|
||||
return _app_settings[charm_name]
|
||||
|
||||
|
||||
def upgrade_ubuntu_lite(from_series='xenial', to_series='bionic'):
|
||||
"""Validate that we can upgrade the ubuntu-lite charm.
|
||||
|
||||
:param from_series: What series are we upgrading from
|
||||
:type from_series: str
|
||||
:param to_series: What series are we upgrading to
|
||||
:type to_series: str
|
||||
"""
|
||||
completed_machines = []
|
||||
asyncio.get_event_loop().run_until_complete(
|
||||
parallel_series_upgrade(
|
||||
'ubuntu-lite', pause_non_leader_primary=False,
|
||||
pause_non_leader_subordinate=False,
|
||||
completed_machines=completed_machines, origin=None)
|
||||
)
|
||||
|
||||
|
||||
async def parallel_series_upgrade(
|
||||
application,
|
||||
from_series='xenial',
|
||||
to_series='bionic',
|
||||
origin='openstack-origin',
|
||||
pause_non_leader_primary=True,
|
||||
pause_non_leader_subordinate=True,
|
||||
pre_upgrade_functions=None,
|
||||
post_upgrade_functions=None,
|
||||
post_application_upgrade_functions=None,
|
||||
completed_machines=None,
|
||||
follower_first=False,
|
||||
files=None,
|
||||
workaround_script=None
|
||||
):
|
||||
"""Perform series upgrade on an application in parallel.
|
||||
|
||||
:param unit_name: Unit Name
|
||||
:type unit_name: str
|
||||
:param machine_num: Machine number
|
||||
:type machine_num: str
|
||||
:param from_series: The series from which to upgrade
|
||||
:type from_series: str
|
||||
:param to_series: The series to which to upgrade
|
||||
:type to_series: str
|
||||
:param origin: The configuration setting variable name for changing origin
|
||||
source. (openstack-origin or source)
|
||||
:type origin: str
|
||||
:param pause_non_leader_primary: Whether the non-leader applications should
|
||||
be paused
|
||||
:type pause_non_leader_primary: bool
|
||||
:param pause_non_leader_subordinate: Whether the non-leader subordinate
|
||||
hacluster applications should be
|
||||
paused
|
||||
:type pause_non_leader_subordinate: bool
|
||||
|
||||
:param pre_upgrade_functions: A list of Zaza functions to call before
|
||||
the upgrade is started on each machine
|
||||
:type pre_upgrade_functions: List[str]
|
||||
:param post_upgrade_functions: A list of Zaza functions to call when
|
||||
the upgrade is complete on each machine
|
||||
:type post_upgrade_functions: List[str]
|
||||
:param post_application_upgrade_functions: A list of Zaza functions
|
||||
to call when the upgrade is complete
|
||||
on all machine in the application
|
||||
:param follower_first: Should the follower(s) be upgraded first
|
||||
:type follower_first: bool
|
||||
:type post_application_upgrade_functions: List[str]
|
||||
:param files: Workaround files to scp to unit under upgrade
|
||||
:type files: list
|
||||
:param workaround_script: Workaround script to run during series upgrade
|
||||
:type workaround_script: str
|
||||
:returns: None
|
||||
:rtype: None
|
||||
"""
|
||||
if completed_machines is None:
|
||||
completed_machines = []
|
||||
if follower_first:
|
||||
logging.error("leader_first is ignored for parallel upgrade")
|
||||
logging.info(
|
||||
"About to upgrade the units of {} in parallel (follower first: {})"
|
||||
.format(application, follower_first))
|
||||
|
||||
status = (await model.async_get_status()).applications[application]
|
||||
logging.info(
|
||||
"Configuring leader / non leaders for {}".format(application))
|
||||
leader, non_leaders = get_leader_and_non_leaders(status)
|
||||
for leader_name, leader_unit in leader.items():
|
||||
leader_machine = leader_unit["machine"]
|
||||
leader = leader_name
|
||||
machines = [
|
||||
unit["machine"] for name, unit
|
||||
in non_leaders.items()
|
||||
if unit['machine'] not in completed_machines]
|
||||
|
||||
await maybe_pause_things(
|
||||
status,
|
||||
non_leaders,
|
||||
pause_non_leader_subordinate,
|
||||
pause_non_leader_primary)
|
||||
await series_upgrade_utils.async_set_series(
|
||||
application, to_series=to_series)
|
||||
app_idle = [
|
||||
model.async_wait_for_unit_idle(unit) for unit in status["units"]
|
||||
]
|
||||
await asyncio.gather(*app_idle)
|
||||
prepare_group = [
|
||||
prepare_series_upgrade(machine, to_series=to_series)
|
||||
for machine in machines]
|
||||
await asyncio.gather(*prepare_group)
|
||||
await prepare_series_upgrade(leader_machine, to_series=to_series)
|
||||
if leader_machine not in completed_machines:
|
||||
machines.append(leader_machine)
|
||||
upgrade_group = [
|
||||
series_upgrade_machine(
|
||||
machine,
|
||||
origin=origin,
|
||||
application=application,
|
||||
files=files, workaround_script=workaround_script,
|
||||
post_upgrade_functions=post_upgrade_functions)
|
||||
for machine in machines
|
||||
]
|
||||
await asyncio.gather(*upgrade_group)
|
||||
completed_machines.extend(machines)
|
||||
await run_post_application_upgrade_functions(
|
||||
post_application_upgrade_functions)
|
||||
|
||||
|
||||
async def serial_series_upgrade(
|
||||
application,
|
||||
from_series='xenial',
|
||||
to_series='bionic',
|
||||
origin='openstack-origin',
|
||||
pause_non_leader_primary=True,
|
||||
pause_non_leader_subordinate=True,
|
||||
pre_upgrade_functions=None,
|
||||
post_upgrade_functions=None,
|
||||
post_application_upgrade_functions=None,
|
||||
completed_machines=None,
|
||||
follower_first=False,
|
||||
files=None,
|
||||
workaround_script=None
|
||||
):
|
||||
"""Perform series upgrade on an application in serial.
|
||||
|
||||
:param unit_name: Unit Name
|
||||
:type unit_name: str
|
||||
:param machine_num: Machine number
|
||||
:type machine_num: str
|
||||
:param from_series: The series from which to upgrade
|
||||
:type from_series: str
|
||||
:param to_series: The series to which to upgrade
|
||||
:type to_series: str
|
||||
:param origin: The configuration setting variable name for changing origin
|
||||
source. (openstack-origin or source)
|
||||
:type origin: str
|
||||
:param pause_non_leader_primary: Whether the non-leader applications should
|
||||
be paused
|
||||
:type pause_non_leader_primary: bool
|
||||
:param pause_non_leader_subordinate: Whether the non-leader subordinate
|
||||
hacluster applications should be
|
||||
paused
|
||||
:type pause_non_leader_subordinate: bool
|
||||
|
||||
:param pre_upgrade_functions: A list of Zaza functions to call before
|
||||
the upgrade is started on each machine
|
||||
:type pre_upgrade_functions: List[str]
|
||||
:param post_upgrade_functions: A list of Zaza functions to call when
|
||||
the upgrade is complete on each machine
|
||||
:type post_upgrade_functions: List[str]
|
||||
:param post_application_upgrade_functions: A list of Zaza functions
|
||||
to call when the upgrade is complete
|
||||
on all machine in the application
|
||||
:param follower_first: Should the follower(s) be upgraded first
|
||||
:type follower_first: bool
|
||||
:type post_application_upgrade_functions: List[str]
|
||||
:param files: Workaround files to scp to unit under upgrade
|
||||
:type files: list
|
||||
:param workaround_script: Workaround script to run during series upgrade
|
||||
:type workaround_script: str
|
||||
:returns: None
|
||||
:rtype: None
|
||||
"""
|
||||
if completed_machines is None:
|
||||
completed_machines = []
|
||||
logging.info(
|
||||
"About to upgrade the units of {} in serial (follower first: {})"
|
||||
.format(application, follower_first))
|
||||
status = (await model.async_get_status()).applications[application]
|
||||
logging.info(
|
||||
"Configuring leader / non leaders for {}".format(application))
|
||||
leader, non_leaders = get_leader_and_non_leaders(status)
|
||||
for leader_name, leader_unit in leader.items():
|
||||
leader_machine = leader_unit["machine"]
|
||||
leader = leader_name
|
||||
|
||||
await maybe_pause_things(
|
||||
status,
|
||||
non_leaders,
|
||||
pause_non_leader_subordinate,
|
||||
pause_non_leader_primary)
|
||||
await series_upgrade_utils.async_set_series(
|
||||
application, to_series=to_series)
|
||||
if not follower_first and leader_machine not in completed_machines:
|
||||
await model.async_wait_for_unit_idle(leader)
|
||||
await prepare_series_upgrade(leader_machine, to_series=to_series)
|
||||
logging.info("About to upgrade leader of {}: {}"
|
||||
.format(application, leader_machine))
|
||||
await series_upgrade_machine(
|
||||
leader_machine,
|
||||
origin=origin,
|
||||
application=application,
|
||||
files=files, workaround_script=workaround_script,
|
||||
post_upgrade_functions=post_upgrade_functions)
|
||||
completed_machines.append(leader_machine)
|
||||
|
||||
# for machine in machines:
|
||||
for unit_name, unit in non_leaders.items():
|
||||
machine = unit['machine']
|
||||
if machine in completed_machines:
|
||||
continue
|
||||
await model.async_wait_for_unit_idle(unit_name)
|
||||
await prepare_series_upgrade(machine, to_series=to_series)
|
||||
logging.info("About to upgrade follower of {}: {}"
|
||||
.format(application, machine))
|
||||
await series_upgrade_machine(
|
||||
machine,
|
||||
origin=origin,
|
||||
application=application,
|
||||
files=files, workaround_script=workaround_script,
|
||||
post_upgrade_functions=post_upgrade_functions)
|
||||
completed_machines.append(machine)
|
||||
|
||||
if follower_first and leader_machine not in completed_machines:
|
||||
await model.async_wait_for_unit_idle(leader)
|
||||
await prepare_series_upgrade(leader_machine, to_series=to_series)
|
||||
logging.info("About to upgrade leader of {}: {}"
|
||||
.format(application, leader_machine))
|
||||
await series_upgrade_machine(
|
||||
leader_machine,
|
||||
origin=origin,
|
||||
application=application,
|
||||
files=files, workaround_script=workaround_script,
|
||||
post_upgrade_functions=post_upgrade_functions)
|
||||
completed_machines.append(leader_machine)
|
||||
await run_post_application_upgrade_functions(
|
||||
post_application_upgrade_functions)
|
||||
|
||||
|
||||
async def series_upgrade_machine(
|
||||
machine,
|
||||
origin=None,
|
||||
application=None,
|
||||
post_upgrade_functions=None,
|
||||
pre_upgrade_functions=None,
|
||||
files=None,
|
||||
workaround_script=None):
|
||||
"""Perform series upgrade on an machine.
|
||||
|
||||
:param machine_num: Machine number
|
||||
:type machine_num: str
|
||||
:param files: Workaround files to scp to unit under upgrade
|
||||
:type files: list
|
||||
:param workaround_script: Workaround script to run during series upgrade
|
||||
:type workaround_script: str
|
||||
:param pre_upgrade_functions: A list of Zaza functions to call before
|
||||
the upgrade is started on each machine
|
||||
:type pre_upgrade_functions: List[str]
|
||||
:param post_upgrade_functions: A list of Zaza functions to call when
|
||||
the upgrade is complete on each machine
|
||||
:type post_upgrade_functions: List[str]
|
||||
:returns: None
|
||||
:rtype: None
|
||||
"""
|
||||
logging.info(
|
||||
"About to series-upgrade ({})".format(machine))
|
||||
await run_pre_upgrade_functions(machine, pre_upgrade_functions)
|
||||
await async_dist_upgrade(machine)
|
||||
await async_do_release_upgrade(machine)
|
||||
await reboot(machine)
|
||||
if origin:
|
||||
await os_utils.async_set_origin(application, origin)
|
||||
await series_upgrade_utils.async_complete_series_upgrade(machine)
|
||||
await run_post_upgrade_functions(post_upgrade_functions)
|
||||
|
||||
|
||||
async def run_pre_upgrade_functions(machine, pre_upgrade_functions):
|
||||
"""Execute list supplied functions.
|
||||
|
||||
Each of the supplied functions will be called with a single
|
||||
argument of the machine that is about to be upgraded.
|
||||
|
||||
:param machine: Machine that is about to be upgraded
|
||||
:type machine: str
|
||||
:param pre_upgrade_functions: List of awaitable functions
|
||||
:type pre_upgrade_functions: [function, function, ...]
|
||||
"""
|
||||
if pre_upgrade_functions:
|
||||
for func in pre_upgrade_functions:
|
||||
logging.info("Running {}".format(func))
|
||||
m = cl_utils.get_class(func)
|
||||
await m(machine)
|
||||
|
||||
|
||||
async def run_post_upgrade_functions(post_upgrade_functions):
|
||||
"""Execute list supplied functions.
|
||||
|
||||
:param post_upgrade_functions: List of awaitable functions
|
||||
:type post_upgrade_functions: [function, function, ...]
|
||||
"""
|
||||
if post_upgrade_functions:
|
||||
for func in post_upgrade_functions:
|
||||
logging.info("Running {}".format(func))
|
||||
m = cl_utils.get_class(func)
|
||||
await m()
|
||||
|
||||
|
||||
async def run_post_application_upgrade_functions(post_upgrade_functions):
|
||||
"""Execute list supplied functions.
|
||||
|
||||
:param post_upgrade_functions: List of awaitable functions
|
||||
:type post_upgrade_functions: [function, function, ...]
|
||||
"""
|
||||
if post_upgrade_functions:
|
||||
for func in post_upgrade_functions:
|
||||
logging.info("Running {}".format(func))
|
||||
m = cl_utils.get_class(func)
|
||||
await m()
|
||||
|
||||
|
||||
async def maybe_pause_things(
|
||||
status, units, pause_non_leader_subordinate=True,
|
||||
pause_non_leader_primary=True):
|
||||
"""Pause the non-leaders, based on the run configuration.
|
||||
|
||||
:param status: Juju status for an application
|
||||
:type status: juju.applications
|
||||
:param units: List of units to paybe pause
|
||||
:type units: LIst[str]
|
||||
:param pause_non_leader_subordinate: Should the non leader
|
||||
subordinate be paused
|
||||
:type pause_non_leader_subordinate: bool
|
||||
:param pause_non_leader_primary: Should the non leaders be paused
|
||||
:type pause_non_leader_primary: bool
|
||||
:returns: Nothing
|
||||
:trype: None
|
||||
"""
|
||||
subordinate_pauses = []
|
||||
leader_pauses = []
|
||||
for unit in units:
|
||||
if pause_non_leader_subordinate:
|
||||
if status["units"][unit].get("subordinates"):
|
||||
for subordinate in status["units"][unit]["subordinates"]:
|
||||
_app = subordinate.split('/')[0]
|
||||
if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST:
|
||||
logging.info("Skipping pausing {} - blacklisted"
|
||||
.format(subordinate))
|
||||
else:
|
||||
logging.info("Pausing {}".format(subordinate))
|
||||
subordinate_pauses.append(model.async_run_action(
|
||||
subordinate, "pause", action_params={}))
|
||||
if pause_non_leader_primary:
|
||||
logging.info("Pausing {}".format(unit))
|
||||
leader_pauses.append(
|
||||
model.async_run_action(unit, "pause", action_params={}))
|
||||
await asyncio.gather(*leader_pauses)
|
||||
await asyncio.gather(*subordinate_pauses)
|
||||
|
||||
|
||||
def get_leader_and_non_leaders(status):
|
||||
"""Get the leader and non-leader Juju units.
|
||||
|
||||
This function returns a tuple that looks like:
|
||||
|
||||
({
|
||||
'unit/1': juju.Unit,
|
||||
},
|
||||
{
|
||||
'unit/0': juju.Unit,
|
||||
'unit/2': juju.unit,
|
||||
})
|
||||
|
||||
The first entry of this tuple is the leader, and the second is
|
||||
all non-leader units.
|
||||
|
||||
:returns: A tuple of dicts identifying leader and non-leaders
|
||||
:rtype: Dict[str, List[juju.Unit]]
|
||||
"""
|
||||
leader = None
|
||||
non_leaders = {}
|
||||
for name, unit in status["units"].items():
|
||||
if unit.get("leader"):
|
||||
leader = {name: unit}
|
||||
else:
|
||||
non_leaders[name] = unit
|
||||
return (leader, non_leaders)
|
||||
|
||||
|
||||
async def prepare_series_upgrade(machine, to_series):
|
||||
"""Execute juju series-upgrade prepare on machine.
|
||||
|
||||
NOTE: This is a new feature in juju behind a feature flag and not yet in
|
||||
libjuju.
|
||||
export JUJU_DEV_FEATURE_FLAGS=upgrade-series
|
||||
:param machine_num: Machine number
|
||||
:type machine_num: str
|
||||
:param to_series: The series to which to upgrade
|
||||
:type to_series: str
|
||||
:returns: None
|
||||
:rtype: None
|
||||
"""
|
||||
logging.info("Preparing series upgrade for: {}".format(machine))
|
||||
await series_upgrade_utils.async_prepare_series_upgrade(
|
||||
machine, to_series=to_series)
|
||||
|
||||
|
||||
async def reboot(machine):
|
||||
"""Reboot the named machine.
|
||||
|
||||
:param machine: Machine to reboot
|
||||
:type machine: str
|
||||
:returns: Nothing
|
||||
:rtype: None
|
||||
"""
|
||||
try:
|
||||
await model.async_run_on_machine(machine, 'sudo init 6 & exit')
|
||||
# await run_on_machine(unit, "sudo reboot && exit")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logging.warn("Error doing reboot: {}".format(e))
|
||||
pass
|
||||
|
||||
|
||||
async def async_dist_upgrade(machine):
|
||||
"""Run dist-upgrade on unit after update package db.
|
||||
|
||||
:param machine: Machine Number
|
||||
:type machine: str
|
||||
:returns: None
|
||||
:rtype: None
|
||||
"""
|
||||
logging.info('Updating package db ' + machine)
|
||||
update_cmd = 'sudo apt-get update'
|
||||
await model.async_run_on_machine(machine, update_cmd)
|
||||
|
||||
logging.info('Updating existing packages ' + machine)
|
||||
dist_upgrade_cmd = (
|
||||
"""yes | sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes """
|
||||
"""-o "Dpkg::Options::=--force-confdef" """
|
||||
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
|
||||
await model.async_run_on_machine(machine, dist_upgrade_cmd)
|
||||
|
||||
|
||||
async def async_do_release_upgrade(machine):
|
||||
"""Run do-release-upgrade noninteractive.
|
||||
|
||||
:param machine: Machine Name
|
||||
:type machine: str
|
||||
:returns: None
|
||||
:rtype: None
|
||||
"""
|
||||
logging.info('Upgrading ' + machine)
|
||||
do_release_upgrade_cmd = (
|
||||
'yes | sudo DEBIAN_FRONTEND=noninteractive '
|
||||
'do-release-upgrade -f DistUpgradeViewNonInteractive')
|
||||
|
||||
await model.async_run_on_machine(
|
||||
machine, do_release_upgrade_cmd, timeout="120m")
|
||||
@@ -687,6 +687,7 @@ async def async_complete_series_upgrade(machine_num):
|
||||
juju_model = await model.async_get_juju_model()
|
||||
cmd = ["juju", "upgrade-series", "-m", juju_model,
|
||||
machine_num, "complete"]
|
||||
logging.info("About to call '{}'".format(cmd))
|
||||
await os_utils.check_call(cmd)
|
||||
|
||||
|
||||
@@ -704,6 +705,7 @@ async def async_set_series(application, to_series):
|
||||
juju_model = await model.async_get_juju_model()
|
||||
cmd = ["juju", "set-series", "-m", juju_model,
|
||||
application, to_series]
|
||||
logging.info("About to call '{}'".format(cmd))
|
||||
await os_utils.check_call(cmd)
|
||||
|
||||
|
||||
|
||||
@@ -20,15 +20,17 @@ import zaza.model
|
||||
|
||||
|
||||
SERVICE_GROUPS = collections.OrderedDict([
|
||||
('Stateful Services', ['percona-cluster', 'rabbitmq-server', 'ceph-mon']),
|
||||
('Core Identity', ['keystone']),
|
||||
('Control Plane', [
|
||||
'aodh', 'barbican', 'ceilometer', 'ceph-mon', 'ceph-fs',
|
||||
'aodh', 'barbican', 'ceilometer', 'ceph-fs',
|
||||
'ceph-radosgw', 'cinder', 'designate',
|
||||
'designate-bind', 'glance', 'gnocchi', 'heat', 'manila',
|
||||
'manila-generic', 'neutron-api', 'neutron-gateway', 'placement',
|
||||
'nova-cloud-controller', 'openstack-dashboard']),
|
||||
('Data Plane', [
|
||||
'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage'])
|
||||
'nova-compute', 'ceph-osd',
|
||||
'swift-proxy', 'swift-storage'])
|
||||
])
|
||||
|
||||
UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster']
|
||||
|
||||
Reference in New Issue
Block a user