From 3efdbe4995df0611394b52d9d248da1c914aa41b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 5 Mar 2019 12:55:49 +0000 Subject: [PATCH] Add masakari test support Add support for testing masakari. --- zaza/charm_tests/masakari/setup.py | 22 ++++ zaza/configure/masakari.py | 169 +++++++++++++++++++++++++++++ zaza/utilities/openstack.py | 23 +++- 3 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 zaza/charm_tests/masakari/setup.py create mode 100644 zaza/configure/masakari.py diff --git a/zaza/charm_tests/masakari/setup.py b/zaza/charm_tests/masakari/setup.py new file mode 100644 index 0000000..53ca1be --- /dev/null +++ b/zaza/charm_tests/masakari/setup.py @@ -0,0 +1,22 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for setting up masakari tests.""" + +import zaza.configure.masakari + + +def create_segments(): + """Create simple segment setup.""" + zaza.configure.masakari.create_segments() diff --git a/zaza/configure/masakari.py b/zaza/configure/masakari.py new file mode 100644 index 0000000..f4f84ac --- /dev/null +++ b/zaza/configure/masakari.py @@ -0,0 +1,169 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configure and manage masakari. + +Functions for managing masakari resources and simulating compute node loss +and recovery. +""" + +import logging + +import zaza.model +import zaza.utilities.openstack as openstack_utils + +ROUND_ROBIN = 'round-robin' + + +def roundrobin_assign_hosts_to_segments(nova_client, masakari_client): + """Assign hypervisors to segments in a round-robin fashion. + + :param nova_client: Authenticated nova client + :type nova_client: novaclient.Client + :param masakari_client: Authenticated masakari client + :type masakari_client: openstack.instance_ha.v1._proxy.Proxy + """ + hypervisors = nova_client.hypervisors.list() + segment_ids = [s.uuid for s in masakari_client.segments()] + segment_ids = segment_ids * len(hypervisors) + for hypervisor in hypervisors: + target_segment = segment_ids.pop() + hostname = hypervisor.hypervisor_hostname.split('.')[0] + logging.info('Adding {} to segment {}'.format(hostname, + target_segment)) + masakari_client.create_host( + name=hostname, + segment_id=target_segment, + recovery_method='auto', + control_attributes='SSH', + type='COMPUTE') + + +HOST_ASSIGNMENT_METHODS = { + ROUND_ROBIN: roundrobin_assign_hosts_to_segments +} + + +def create_segments(segment_number=1, host_assignment_method=None): + """Create a masakari segment and populate it with hypervisors. + + :param segment_number: Number of segments to create + :type segment_number: int + :param host_assignment_method: Method to use to assign hypervisors to + segments + :type host_assignment_method: f() + """ + host_assignment_method = host_assignment_method or ROUND_ROBIN + keystone_session = openstack_utils.get_overcloud_keystone_session() + nova_client = openstack_utils.get_nova_session_client(keystone_session) + masakari_client = openstack_utils.get_masakari_session_client( + keystone_session) + for segment_number in range(0, segment_number): + segment_name = 'seg{}'.format(segment_number) + logging.info('Creating segment {}'.format(segment_name)) + masakari_client.create_segment( + name=segment_name, + recovery_method='auto', + service_type='COMPUTE') + HOST_ASSIGNMENT_METHODS[host_assignment_method]( + nova_client, + masakari_client) + + +def enable_hosts(masakari_client=None): + """Enable all hypervisors within masakari. + + Enable all hosts across all segments within masakari. This does not + enable the hypervisor from a nova POV. + + :param masakari_client: Authenticated masakari client + :type masakari_client: openstack.instance_ha.v1._proxy.Proxy + """ + if not masakari_client: + keystone_session = openstack_utils.get_overcloud_keystone_session() + masakari_client = openstack_utils.get_masakari_session_client( + keystone_session) + + for segment in masakari_client.segments(): + for host in masakari_client.hosts(segment_id=segment.uuid): + if host.on_maintenance: + logging.info("Removing maintenance mode from masakari " + "host {}".format(host.uuid)) + masakari_client.update_host( + host.uuid, + segment_id=segment.uuid, + **{'on_maintenance': False}) + + +def _svc_control(unit_name, action, services, model_name): + """Enable/Disable services on a unit. + + This is a simplistic method for controlling services, hence its private. + + :param unit_name: Juju name of unit (app/n) + :type unit_name: str + :param action: systemctl action to perform on unit (start/stop etc) + :type action: str + :param services: List of services to perform action against + :type services: [] + :param model_name: Name of model unit_name resides in. + :type model_name: str + """ + logging.info('{} {} on {}'.format(action.title(), services, unit_name)) + cmds = [] + for svc in services: + cmds.append("systemctl {} {}".format(action, svc)) + zaza.model.run_on_unit( + unit_name, command=';'.join(cmds), + model_name=model_name) + + +def simulate_compute_host_failure(unit_name, model_name): + """Simulate compute node failure from a masakari and nova POV. + + Masakari uses corosync/pacemaker to detect failure and nova check + nova-compute. Shutting down these services causes masakari and nova to + mark them as down. + + :param unit_name: Juju name of unit (app/n) + :type unit_name: str + :param model_name: Name of model unit_name resides in. + :type model_name: str + """ + logging.info('Simulating failure of compute node {}'.format(unit_name)) + _svc_control( + unit_name, + 'stop', + ['corosync', 'pacemaker', 'nova-compute'], + model_name) + + +def simulate_compute_host_recovery(unit_name, model_name): + """Simulate compute node recovery from a masakari and nova POV. + + Masakari uses corosync/pacemaker to detect failure and nova check + nova-compute. Starting these services is a prerequisite to marking + them as recovered. + + :param unit_name: Juju name of unit (app/n) + :type unit_name: str + :param model_name: Name of model unit_name resides in. + :type model_name: str + """ + logging.info('Simulating recovery of compute node {}'.format(unit_name)) + _svc_control( + unit_name, + 'start', + ['corosync', 'pacemaker', 'nova-compute'], + model_name) diff --git a/zaza/utilities/openstack.py b/zaza/utilities/openstack.py index 2a71157..3d45cae 100644 --- a/zaza/utilities/openstack.py +++ b/zaza/utilities/openstack.py @@ -23,6 +23,8 @@ from .os_versions import ( OPENSTACK_RELEASES_PAIRS, ) +from openstack import connection + from cinderclient import client as cinderclient from glanceclient import Client as GlanceClient @@ -215,7 +217,7 @@ def get_swift_session_client(session): def get_octavia_session_client(session, service_type='load-balancer', interface='internal'): - """Return neutronclient authenticated by keystone session. + """Return octavia client authenticated by keystone session. :param session: Keystone session object :type session: keystoneauth1.session.Session object @@ -251,6 +253,25 @@ def get_cinder_session_client(session, version=2): return cinderclient.Client(session=session, version=version) +def get_masakari_session_client(session, interface='internal', + region_name='RegionOne'): + """Return masakari client authenticated by keystone session. + + :param session: Keystone session object + :type session: keystoneauth1.session.Session object + :param interface: Interface to look for in catalog + :type interface: str + :param region_name: Region name to use in catalogue lookup + :type region_name: str + :returns: Authenticated masakari client + :rtype: openstack.instance_ha.v1._proxy.Proxy + """ + conn = connection.Connection(session=session, + interface=interface, + region_name=region_name) + return conn.instance_ha + + def get_keystone_scope(): """Return Keystone scope based on OpenStack release of the overcloud.