arif/zaza-openstack-tests
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tidy up and add more debugability

Browse Source
This commit is contained in:
Chris MacNaughton
2020-04-15 09:40:37 +02:00
parent 6c70ee5171
commit 592622935c
4 changed files with 43 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py
+11 -4
View File
@@ -177,6 +177,9 @@ class TestParallelSeriesUpgrade(AioTestCase):
self.async_run_action = mock.AsyncMock()
self.model.async_run_action = self.async_run_action
self.async_block_until = mock.AsyncMock()
self.model.async_block_until = self.async_block_until
@mock.patch.object(upgrade_utils.cl_utils, 'get_class')
async def test_run_post_application_upgrade_functions(
self,
@@ -225,11 +228,14 @@ class TestParallelSeriesUpgrade(AioTestCase):
mock_async_set_series.assert_called_once_with(
'mongodb', to_series='xenial')
self.juju_status.assert_called()
# The below is using `any_order=True` because the ordering is
# undetermined and differs between python versions
mock_async_prepare_series_upgrade.assert_has_calls([
mock.call('1', to_series='xenial'),
mock.call('2', to_series='xenial'),
mock.call('0', to_series='xenial'),
])
], any_order=True)
mock_maybe_pause_things.assert_called()
mock_series_upgrade_machine.assert_has_calls([
mock.call(
@@ -495,7 +501,7 @@ class TestParallelSeriesUpgrade(AioTestCase):
async def test_reboot(self, mock_run_on_machine):
await upgrade_utils.reboot('1')
mock_run_on_machine.assert_called_once_with(
'1', 'shutdown --reboot now & exit'
'1', 'sudo init 6 & exit'
)
async def test_run_on_machine(self):
@@ -517,10 +523,11 @@ class TestParallelSeriesUpgrade(AioTestCase):
async def test_async_dist_upgrade(self, mock_run_on_machine):
await upgrade_utils.async_dist_upgrade('1')
apt_update_command = (
"""yes | sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """
"""yes | sudo DEBIAN_FRONTEND=noninteractive """
"""apt-get --assume-yes """
"""-o "Dpkg::Options::=--force-confdef" """
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
mock_run_on_machine.assert_has_calls([
mock.call('1', 'sudo apt update'),
mock.call('1', 'sudo apt-get update'),
mock.call('1', apt_update_command),
])
zaza/openstack/utilities/generic.py
+7
View File
@@ -385,10 +385,17 @@ async def check_call(cmd):
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate()
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
if proc.returncode != 0:
logging.warn("STDOUT: {}".format(stdout))
logging.warn("STDERR: {}".format(stderr))
raise subprocess.CalledProcessError(proc.returncode, cmd)
else:
if stderr:
logging.info("STDERR: {} ({})".format(stderr, ' '.join(cmd)))
if stdout:
logging.info("STDOUT: {} ({})".format(stdout, ' '.join(cmd)))
def set_dpkg_non_interactive_on_unit(
zaza/openstack/utilities/parallel_series_upgrade.py
+23 -30
View File
@@ -214,12 +214,10 @@ async def parallel_series_upgrade(
]
await asyncio.gather(*app_idle)
prepare_group = [
series_upgrade_utils.async_prepare_series_upgrade(
machine, to_series=to_series)
prepare_series_upgrade(machine, to_series=to_series)
for machine in machines]
await asyncio.gather(*prepare_group)
await series_upgrade_utils.async_prepare_series_upgrade(
leader_machine, to_series=to_series)
await prepare_series_upgrade(leader_machine, to_series=to_series)
if leader_machine not in completed_machines:
machines.append(leader_machine)
upgrade_group = [
@@ -231,7 +229,8 @@ async def parallel_series_upgrade(
]
await asyncio.gather(*upgrade_group)
completed_machines.extend(machines)
run_post_application_upgrade_functions(post_application_upgrade_functions)
await run_post_application_upgrade_functions(
post_application_upgrade_functions)
async def serial_series_upgrade(
@@ -302,11 +301,6 @@ async def serial_series_upgrade(
leader_machine = leader_unit["machine"]
leader = leader_name
machines = [
unit["machine"] for name, unit
in non_leaders.items()
if unit['machine'] not in completed_machines]
await maybe_pause_things(
status,
non_leaders,
@@ -318,8 +312,7 @@ async def serial_series_upgrade(
await os_utils.async_set_origin(application, origin)
if not follower_first and leader_machine not in completed_machines:
await wait_for_unit_idle(leader)
await series_upgrade_utils.async_prepare_series_upgrade(
leader_machine, to_series=to_series)
await prepare_series_upgrade(leader_machine, to_series=to_series)
logging.info("About to upgrade leader of {}: {}"
.format(application, leader_machine))
await series_upgrade_machine(
@@ -334,8 +327,7 @@ async def serial_series_upgrade(
if machine in completed_machines:
continue
await wait_for_unit_idle(unit_name)
await series_upgrade_utils.async_prepare_series_upgrade(
machine, to_series=to_series)
await prepare_series_upgrade(machine, to_series=to_series)
logging.info("About to upgrade follower of {}: {}"
.format(application, machine))
await series_upgrade_machine(
@@ -346,8 +338,7 @@ async def serial_series_upgrade(
if follower_first and leader_machine not in completed_machines:
await wait_for_unit_idle(leader)
await series_upgrade_utils.async_prepare_series_upgrade(
leader_machine, to_series=to_series)
await prepare_series_upgrade(leader_machine, to_series=to_series)
logging.info("About to upgrade leader of {}: {}"
.format(application, leader_machine))
await series_upgrade_machine(
@@ -383,9 +374,8 @@ async def series_upgrade_machine(
:rtype: None
"""
logging.info(
"About to dist-upgrade ({})".format(machine))
run_pre_upgrade_functions(machine, pre_upgrade_functions)
"About to series-upgrade ({})".format(machine))
await run_pre_upgrade_functions(machine, pre_upgrade_functions)
await async_dist_upgrade(machine)
await async_do_release_upgrade(machine)
await reboot(machine)
@@ -401,37 +391,40 @@ async def run_pre_upgrade_functions(machine, pre_upgrade_functions):
:param machine: Machine that is about to be upgraded
:type machine: str
:param pre_upgrade_functions: List of functions
:param pre_upgrade_functions: List of awaitable functions
:type pre_upgrade_functions: [function, function, ...]
"""
if pre_upgrade_functions:
for func in pre_upgrade_functions:
logging.info("Running {}".format(func))
await cl_utils.get_class(func)(machine)
m = cl_utils.get_class(func)
await m(machine)
async def run_post_upgrade_functions(post_upgrade_functions):
"""Execute list supplied functions.
:param post_upgrade_functions: List of functions
:param post_upgrade_functions: List of awaitable functions
:type post_upgrade_functions: [function, function, ...]
"""
if post_upgrade_functions:
for func in post_upgrade_functions:
logging.info("Running {}".format(func))
await cl_utils.get_class(func)()
m = cl_utils.get_class(func)
await m()
async def run_post_application_upgrade_functions(post_upgrade_functions):
"""Execute list supplied functions.
:param post_upgrade_functions: List of functions
:param post_upgrade_functions: List of awaitable functions
:type post_upgrade_functions: [function, function, ...]
"""
if post_upgrade_functions:
for func in post_upgrade_functions:
logging.info("Running {}".format(func))
await cl_utils.get_class(func)()
m = cl_utils.get_class(func)
await m()
async def maybe_pause_things(
@@ -515,7 +508,7 @@ async def prepare_series_upgrade(machine, to_series):
:returns: None
:rtype: None
"""
logging.debug("Preparing series upgrade for: {}".format(machine))
logging.info("Preparing series upgrade for: {}".format(machine))
await series_upgrade_utils.async_prepare_series_upgrade(
machine, to_series=to_series)
@@ -529,7 +522,7 @@ async def reboot(machine):
:rtype: None
"""
try:
await run_on_machine(machine, 'shutdown --reboot now & exit')
await run_on_machine(machine, 'sudo init 6 & exit')
# await run_on_machine(unit, "sudo reboot && exit")
except subprocess.CalledProcessError as e:
logging.warn("Error doing reboot: {}".format(e))
@@ -556,7 +549,7 @@ async def run_on_machine(machine, command, model_name=None, timeout=None):
if timeout:
cmd.append('--timeout={}'.format(timeout))
cmd.append(command)
logging.debug("About to call '{}'".format(cmd))
logging.info("About to call '{}'".format(cmd))
await os_utils.check_call(cmd)
@@ -569,12 +562,12 @@ async def async_dist_upgrade(machine):
:rtype: None
"""
logging.info('Updating package db ' + machine)
update_cmd = 'sudo apt update'
update_cmd = 'sudo apt-get update'
await run_on_machine(machine, update_cmd)
logging.info('Updating existing packages ' + machine)
dist_upgrade_cmd = (
"""yes | sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """
"""yes | sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes """
"""-o "Dpkg::Options::=--force-confdef" """
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
await run_on_machine(machine, dist_upgrade_cmd)
zaza/openstack/utilities/series_upgrade.py
+2
View File
@@ -687,6 +687,7 @@ async def async_complete_series_upgrade(machine_num):
juju_model = await model.async_get_juju_model()
cmd = ["juju", "upgrade-series", "-m", juju_model,
machine_num, "complete"]
logging.info("About to call '{}'".format(cmd))
await os_utils.check_call(cmd)
@@ -704,6 +705,7 @@ async def async_set_series(application, to_series):
juju_model = await model.async_get_juju_model()
cmd = ["juju", "set-series", "-m", juju_model,
application, to_series]
logging.info("About to call '{}'".format(cmd))
await os_utils.check_call(cmd)