701e751398
On a recent mosci run the guest used for a restart test had a pending power task which stopped the restart test working properly. This PR adds an assertion that the guest is ready.
231 lines
8.9 KiB
Python
231 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# Copyright 2019 Canonical Ltd.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Encapsulate masakari testing."""
|
|
|
|
from datetime import datetime
|
|
import logging
|
|
import unittest
|
|
import tenacity
|
|
|
|
import novaclient
|
|
|
|
import zaza.model
|
|
import zaza.openstack.charm_tests.test_utils as test_utils
|
|
import zaza.openstack.utilities.juju as juju_utils
|
|
import zaza.openstack.utilities.openstack as openstack_utils
|
|
import zaza.openstack.configure.guest
|
|
import zaza.openstack.configure.hacluster
|
|
import zaza.openstack.configure.masakari
|
|
|
|
|
|
class MasakariTest(test_utils.OpenStackBaseTest):
|
|
"""Encapsulate Masakari tests."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Run class setup for running tests."""
|
|
super(MasakariTest, cls).setUpClass(application_name="masakari")
|
|
cls.current_release = openstack_utils.get_os_release()
|
|
cls.keystone_session = openstack_utils.get_overcloud_keystone_session()
|
|
cls.model_name = zaza.model.get_juju_model()
|
|
cls.nova_client = openstack_utils.get_nova_session_client(
|
|
cls.keystone_session)
|
|
|
|
@classmethod
|
|
def tearDown(cls):
|
|
"""Bring hypervisors and services back up."""
|
|
logging.info('Running teardown')
|
|
for unit in zaza.model.get_units('nova-compute',
|
|
model_name=cls.model_name):
|
|
zaza.openstack.configure.masakari.simulate_compute_host_recovery(
|
|
unit.entity_id,
|
|
model_name=cls.model_name)
|
|
openstack_utils.enable_all_nova_services(cls.nova_client)
|
|
zaza.openstack.configure.masakari.enable_hosts()
|
|
|
|
def ensure_guest(self, vm_name):
|
|
"""Return the existing guest or boot a new one.
|
|
|
|
:param vm_name: Name of guest to lookup
|
|
:type vm_name: str
|
|
:returns: Guest matching name.
|
|
:rtype: novaclient.v2.servers.Server
|
|
"""
|
|
try:
|
|
guest = self.nova_client.servers.find(name=vm_name)
|
|
logging.info('Found existing guest')
|
|
except novaclient.exceptions.NotFound:
|
|
logging.info('Launching new guest')
|
|
guest = zaza.openstack.configure.guest.launch_instance(
|
|
'bionic',
|
|
use_boot_volume=True,
|
|
meta={'HA_Enabled': 'True'},
|
|
vm_name=vm_name)
|
|
return guest
|
|
|
|
def get_guests_compute_info(self, vm_name):
|
|
"""Return the hostname & juju unit of compute host hosting vm.
|
|
|
|
:param vm_name: Name of guest to lookup
|
|
:type vm_name: str
|
|
:returns: Hypervisor name and juju unit name
|
|
:rtype: (str, str)
|
|
"""
|
|
current_hypervisor = openstack_utils.get_hypervisor_for_guest(
|
|
self.nova_client,
|
|
vm_name)
|
|
unit_name = juju_utils.get_unit_name_from_host_name(
|
|
current_hypervisor,
|
|
'nova-compute')
|
|
return current_hypervisor, unit_name
|
|
|
|
def get_guest_qemu_pid(self, compute_unit_name, vm_uuid, model_name=None):
|
|
"""Return the qemu pid of process running guest.
|
|
|
|
:param compute_unit_name: Juju unit name of hypervisor running guest
|
|
:type compute_unit_name: str
|
|
:param vm_uuid: Guests UUID
|
|
:type vm_uuid: str
|
|
:param model_name: Name of model running cloud.
|
|
:type model_name: str
|
|
:returns: PID of qemu process
|
|
:rtype: int
|
|
:raises: ValueError
|
|
"""
|
|
pid_find_cmd = 'pgrep -u libvirt-qemu -f {}'.format(vm_uuid)
|
|
out = zaza.model.run_on_unit(
|
|
compute_unit_name,
|
|
pid_find_cmd,
|
|
model_name=self.model_name)
|
|
return int(out['Stdout'].strip())
|
|
|
|
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60),
|
|
reraise=True, stop=tenacity.stop_after_attempt(5),
|
|
retry=tenacity.retry_if_exception_type(ValueError))
|
|
def wait_for_guest_pid(self, compute_unit_name, vm_uuid, model_name=None):
|
|
"""Wait for the qemu process running guest to appear & return its pid.
|
|
|
|
:param compute_unit_name: Juju unit name of hypervisor running guest
|
|
:type compute_unit_name: str
|
|
:param vm_uuid: Guests UUID
|
|
:type vm_uuid: str
|
|
:param model_name: Name of model running cloud.
|
|
:type model_name: str
|
|
:returns: PID of qemu process
|
|
:rtype: int
|
|
:raises: ValueError
|
|
"""
|
|
return self.get_guest_qemu_pid(
|
|
compute_unit_name,
|
|
vm_uuid,
|
|
model_name=self.model_name)
|
|
|
|
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60),
|
|
reraise=True, stop=tenacity.stop_after_attempt(5),
|
|
retry=tenacity.retry_if_exception_type(AssertionError))
|
|
def wait_for_guest_ready(self, vm_name):
|
|
"""Wait for the guest to be ready.
|
|
|
|
:param vm_name: Name of guest to check.
|
|
:type vm_name: str
|
|
"""
|
|
guest_ready_attr_checks = [
|
|
('OS-EXT-STS:task_state', None),
|
|
('status', 'ACTIVE'),
|
|
('OS-EXT-STS:power_state', 1),
|
|
('OS-EXT-STS:vm_state', 'active')]
|
|
guest = self.nova_client.servers.find(name=vm_name)
|
|
logging.info('Checking guest {} attributes'.format(vm_name))
|
|
for (attr, required_state) in guest_ready_attr_checks:
|
|
logging.info('Checking {} is {}'.format(attr, required_state))
|
|
assert getattr(guest, attr) == required_state
|
|
|
|
def test_instance_failover(self):
|
|
"""Test masakari managed guest migration."""
|
|
# Workaround for Bug #1874719
|
|
zaza.openstack.configure.hacluster.remove_node(
|
|
'masakari',
|
|
'node1')
|
|
# Launch guest
|
|
self.assertTrue(
|
|
zaza.openstack.configure.hacluster.check_all_nodes_online(
|
|
'masakari'))
|
|
vm_name = 'zaza-test-instance-failover'
|
|
self.ensure_guest(vm_name)
|
|
|
|
# Locate hypervisor hosting guest and shut it down
|
|
current_hypervisor, unit_name = self.get_guests_compute_info(vm_name)
|
|
zaza.openstack.configure.masakari.simulate_compute_host_failure(
|
|
unit_name,
|
|
model_name=self.model_name)
|
|
|
|
# Wait for instance move
|
|
logging.info('Waiting for guest to move away from {}'.format(
|
|
current_hypervisor))
|
|
# wait_for_server_migration will throw an exception if migration fails
|
|
openstack_utils.wait_for_server_migration(
|
|
self.nova_client,
|
|
vm_name,
|
|
current_hypervisor)
|
|
|
|
# Bring things back
|
|
zaza.openstack.configure.masakari.simulate_compute_host_recovery(
|
|
unit_name,
|
|
model_name=self.model_name)
|
|
openstack_utils.enable_all_nova_services(self.nova_client)
|
|
zaza.openstack.configure.masakari.enable_hosts()
|
|
self.wait_for_guest_ready(vm_name)
|
|
|
|
def test_instance_restart_on_fail(self):
|
|
"""Test single guest crash and recovery."""
|
|
if self.current_release < openstack_utils.get_os_release(
|
|
'bionic_ussuri'):
|
|
raise unittest.SkipTest(
|
|
"Not supported on {}. Bug #1866638".format(
|
|
self.current_release))
|
|
vm_name = 'zaza-test-instance-failover'
|
|
vm = self.ensure_guest(vm_name)
|
|
self.wait_for_guest_ready(vm_name)
|
|
_, unit_name = self.get_guests_compute_info(vm_name)
|
|
logging.info('{} is running on {}'.format(vm_name, unit_name))
|
|
guest_pid = self.get_guest_qemu_pid(
|
|
unit_name,
|
|
vm.id,
|
|
model_name=self.model_name)
|
|
logging.info('{} pid is {}'.format(vm_name, guest_pid))
|
|
inital_update_time = datetime.strptime(
|
|
vm.updated,
|
|
"%Y-%m-%dT%H:%M:%SZ")
|
|
logging.info('Simulating vm crash of {}'.format(vm_name))
|
|
zaza.openstack.configure.masakari.simulate_guest_crash(
|
|
guest_pid,
|
|
unit_name,
|
|
model_name=self.model_name)
|
|
logging.info('Waiting for {} to be updated and become active'.format(
|
|
vm_name))
|
|
openstack_utils.wait_for_server_update_and_active(
|
|
self.nova_client,
|
|
vm_name,
|
|
inital_update_time)
|
|
new_guest_pid = self.wait_for_guest_pid(
|
|
unit_name,
|
|
vm.id,
|
|
model_name=self.model_name)
|
|
logging.info('{} pid is now {}'.format(vm_name, new_guest_pid))
|
|
assert new_guest_pid and new_guest_pid != guest_pid, (
|
|
"Restart failed or never happened")
|